Add datasource ID

PIPELINES-AVID.rst

@@ -1,11 +1,13 @@
-.. list-table:: Pipeline AVID Mapping
+.. list-table:: Pipeline Advisory UID Mapping
    :header-rows: 1
    :widths: 35 65
 
    * - pipeline name
-     - AVID
+     - Advisory UID
+     - datasource name
    * - alpine_linux_importer_v2
      - {package_name}/{distroversion}/{version}/{vulnerability_id}
+     - alpine_linux
    * - aosp_dataset_fix_commits
      - CVE ID of the record
    * - apache_httpd_importer_v2

vulnerabilities/migrations/0130_advisoryv2_pipeline_id.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.2.11 on 2026-05-18 08:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0129_advisorypoc"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="pipeline_id",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2",
+                max_length=200,
+                null=True,
+            ),
+        ),
+    ]

vulnerabilities/migrations/0131_auto_20260518_0854.py

@@ -0,0 +1,42 @@
+# Generated by Django 5.2.11 on 2026-05-18 08:54
+
+from django.db import migrations, models
+from django.db.models import F
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0130_advisoryv2_pipeline_id"),
+    ]
+
+    def populate_pipeline_id(apps, schema_editor):
+        Advisory = apps.get_model("vulnerabilities", "AdvisoryV2")
+
+        Advisory.objects.update(
+            pipeline_id=F("datasource_id")
+        )
+
+        assert not Advisory.objects.filter(pipeline_id="").exists(), "Some advisories have an empty pipeline_id after the update"
+
+    operations = [
+        migrations.RunPython(populate_pipeline_id, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="pipeline_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2",
+                max_length=200,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="datasource_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the datasource used for this advisory .e.g.: nginx",
+                max_length=200,
+            ),
+        ),
+    ]

vulnerabilities/migrations/0132_migrate_advisoryv2_datasource_ids.py

@@ -0,0 +1,77 @@
+from django.db import migrations
+from django.db.models import F
+from django.db.models import Value
+from django.db.models.functions import Concat
+
+
+def migrate_advisoryv2_datasource_ids(apps, schema_editor):
+    """
+    v2 importers previously stored pipeline_id as datasource_id on AdvisoryV2.
+    Migration 0131 copied that value into pipeline_id; update datasource_id (and avid)
+    to each pipeline's datasource_id, matching rows by pipeline_id.
+    """
+    from vulnerabilities.importers import IMPORTERS_REGISTRY
+    from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+
+    Advisory = apps.get_model("vulnerabilities", "AdvisoryV2")
+
+    for pipeline_class in IMPORTERS_REGISTRY.values():
+        if not issubclass(pipeline_class, VulnerableCodeBaseImporterPipelineV2):
+            continue
+
+        pipeline_id = pipeline_class.pipeline_id
+        datasource_id = pipeline_class.datasource_id
+        if not pipeline_id or not datasource_id:
+            continue
+        if pipeline_id == datasource_id:
+            continue
+
+        Advisory.objects.filter(
+            pipeline_id=pipeline_id,
+            datasource_id=pipeline_id,
+        ).update(
+            datasource_id=datasource_id,
+            avid=Concat(Value(f"{datasource_id}/"), F("advisory_id")),
+        )
+
+
+def reverse_migrate_advisoryv2_datasource_ids(apps, schema_editor):
+    from vulnerabilities.importers import IMPORTERS_REGISTRY
+    from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+
+    Advisory = apps.get_model("vulnerabilities", "AdvisoryV2")
+
+    for pipeline_class in IMPORTERS_REGISTRY.values():
+        if not issubclass(pipeline_class, VulnerableCodeBaseImporterPipelineV2):
+            continue
+        if "v2_importers" not in pipeline_class.__module__:
+            continue
+
+        pipeline_id = pipeline_class.pipeline_id
+        datasource_id = pipeline_class.datasource_id
+        if not pipeline_id or not datasource_id:
+            continue
+        if pipeline_id == datasource_id:
+            continue
+
+        Advisory.objects.filter(
+            pipeline_id=pipeline_id,
+            datasource_id=datasource_id,
+        ).update(
+            datasource_id=pipeline_id,
+            avid=Concat(Value(f"{pipeline_id}/"), F("advisory_id")),
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0131_auto_20260518_0854"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            migrate_advisoryv2_datasource_ids,
+            reverse_code=reverse_migrate_advisoryv2_datasource_ids,
+        ),
+    ]

vulnerabilities/models.py

@@ -3058,7 +3058,15 @@ class AdvisoryV2(models.Model):
         blank=False,
         null=False,
         db_index=True,
-        help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx_importer_v2",
+        help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx",
+    )
+
+    pipeline_id = models.CharField(
+        max_length=200,
+        blank=False,
+        null=False,
+        db_index=True,
+        help_text="Unique ID for the pipeline used for this advisory ." "e.g.: nginx_importer_v2",
     )
 
     # This is similar to a name

vulnerabilities/pipelines/__init__.py

@@ -273,7 +273,7 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
 
     pipeline_id = None  # Unique Pipeline ID, this should be the name of pipeline module.
     license_url = None
-    datasource_name = None
+    datasource_id = None
     spdx_license_expression = None
     repo_url = None
     ignorable_versions = []
@@ -319,6 +319,9 @@ def advisories_count(self) -> int:
         raise NotImplementedError
 
     def collect_and_store_advisories(self):
+        if not self.pipeline_id and not self.datasource_id:
+            self.log("Pipeline must have a unique pipeline_id or datasource_id defined.")
+            return
         collected_advisory_count = 0
         estimated_advisory_count = self.advisories_count()
 
@@ -338,6 +341,7 @@ def collect_and_store_advisories(self):
                 if _obj := insert_advisory_v2(
                     advisory=advisory,
                     pipeline_id=self.pipeline_id,
+                    datasource_id=self.datasource_id,
                     logger=self.log,
                     precedence=self.precedence,
                 ):

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -38,6 +38,7 @@ class AlpineLinuxImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
 
     pipeline_id = "alpine_linux_importer_v2"
     spdx_license_expression = "CC-BY-SA-4.0"
+    datasource_id = "alpine_linux"
     license_url = "https://secdb.alpinelinux.org/license.txt"
     repo_url = "git+https://github.com/aboutcode-org/aboutcode-mirror-alpine-secdb/"
 

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -28,6 +28,7 @@ class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
 
     pipeline_id = "aosp_dataset_fix_commits"
+    datasource_id = "aosp_dataset"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE"
 

vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py

@@ -148,6 +148,7 @@ class ApacheHTTPDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
 
     pipeline_id = "apache_httpd_importer_v2"
+    datasource_id = "apache_httpd"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://www.apache.org/licenses/LICENSE-2.0"
     base_url = "https://httpd.apache.org/security/json/"

vulnerabilities/pipelines/v2_importers/apache_kafka_importer.py

@@ -33,6 +33,7 @@ class ApacheKafkaImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """Import Apache Kafka Advisories"""
 
     pipeline_id = "apache_kafka_importer_v2"
+    datasource_id = "apache_kafka"
     spdx_license_expression = "Apache-2.0"
     importer_name = "Apache Kafka Importer V2"
 

vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py

@@ -36,6 +36,7 @@ class ApacheTomcatImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
 
     pipeline_id = "apache_tomcat_importer_v2"
+    datasource_id = "apache_tomcat"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://www.apache.org/licenses/LICENSE-2.0"
     base_url = "https://tomcat.apache.org/security"

vulnerabilities/pipelines/v2_importers/archlinux_importer.py

@@ -29,6 +29,7 @@ class ArchLinuxImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
 
     pipeline_id = "archlinux_importer_v2"
     spdx_license_expression = "MIT"
+    datasource_id = "archlinux"
     license_url = "https://github.com/archlinux/arch-security-tracker/blob/master/LICENSE"
 
     precedence = 200