fully_imported check on Version manager (#1995)

libraries/github.py

@@ -506,7 +506,6 @@ def handle_commit(commit: ParsedCommit):
                 )
 
             except KeyError:
-                logger.warning(f"KeyError {commit.version=}")
                 return None
 
         def handle_version_diff_stat(diff: VersionDiffStat):

libraries/management/commands/import_library_version_docs_urls.py

@@ -47,12 +47,15 @@ def command(release: str, new: bool, min_version: str):
             processed.
     """
     click.secho("Saving links to version-specific library docs...", fg="green")
-    min_version = f"boost-{min_version}"
-    version_qs = Version.objects.active().filter(name__gte=min_version)
+    version_qs = (
+        Version.objects.with_partials()
+        .active()
+        .filter(name__gte=f"boost-{min_version}")
+    )
     if release:
-        versions = Version.objects.filter(name__icontains=release).order_by("-name")
+        versions = version_qs.filter(name__icontains=release).order_by("-name")
     elif new:
-        versions = [Version.objects.most_recent()]
+        versions = [version_qs.most_recent()]
     else:
         versions = version_qs.order_by("-name")
 

libraries/management/commands/import_library_versions.py

@@ -43,13 +43,15 @@ def command(min_release: str, release: str, new: bool, token: str):
         Overridden by --release if provided.
     """
     click.secho("Saving library-version relationships...", fg="green")
-
-    min_release = f"boost-{min_release}"
-    versions_qs = Version.objects.active().filter(name__gte=min_release)
+    versions_qs = (
+        Version.objects.with_partials()
+        .active()
+        .filter(name__gte=f"boost-{min_release}")
+    )
     if release:
         versions = versions_qs.filter(name__icontains=release).order_by("-name")
     elif new:
-        versions = [Version.objects.most_recent()]
+        versions = [versions_qs.most_recent()]
     else:
         versions = versions_qs.order_by("-name")
 

libraries/management/commands/release_tasks.py

@@ -44,6 +44,7 @@ def set_tasks(self):
                 "Importing most recent beta version",
                 ["import_beta_release", "--delete-versions"],
             ),
+            Action("Importing development versions", ["import_development_versions"]),
             Action("Importing libraries", ["update_libraries"]),
             Action(
                 "Saving library-version relationships", self.import_library_versions
@@ -64,7 +65,7 @@ def set_tasks(self):
 
     def import_versions(self):
         call_command("import_versions")
-        self.latest_version = Version.objects.most_recent()
+        self.latest_version = Version.objects.with_partials().most_recent()
 
     def import_library_versions(self):
         latest_version_number = self.latest_version.name.lstrip("boost-")

libraries/tasks.py

@@ -47,7 +47,7 @@
 @app.task
 def update_library_version_documentation_urls_all_versions():
     """Run the task to update all documentation URLs for all versions"""
-    for version in Version.objects.all().order_by("-name"):
+    for version in Version.objects.with_partials().all().order_by("-name"):
         get_and_store_library_version_documentation_urls_for_version(version.pk)
 
 
@@ -68,7 +68,7 @@ def get_and_store_library_version_documentation_urls_for_version(version_pk):
     database.
     """
     try:
-        version = Version.objects.get(pk=version_pk)
+        version = Version.objects.with_partials().get(pk=version_pk)
     except Version.DoesNotExist:
         logger.error(f"Version does not exist for {version_pk=}")
         raise

libraries/tests/test_models.py

@@ -77,7 +77,9 @@ def test_library_version_multiple_versions(library, library_version):
     assert library.versions.filter(
         library_version__version=library_version.version
     ).exists()
-    other_version = baker.make("versions.Version", name="New Version")
+    other_version = baker.make(
+        "versions.Version", name="New Version", fully_imported=True
+    )
     baker.make("libraries.LibraryVersion", library=library, version=other_version)
     assert library.versions.count() == 2
     assert library.versions.filter(

mailing_list/management/commands/sync_mailinglist_stats.py

@@ -58,12 +58,16 @@ def bulk_create(rows, version):
 
     versions = Version.objects.minor_versions().order_by("version_array")
     columns = ["email", "name", "count"]
-    versions = list(versions) + [Version.objects.get(name="master")]
+    versions = list(versions) + [Version.objects.with_partials().get(name="master")]
     for a, b in pairwise(versions):
         start = a.release_date
         end = b.release_date or date.today()
         if not (start and end):
-            raise ValueError("All x.x.0 versions must have a release date.")
+            msg = (
+                "All x.x.0 versions must have a release date."
+                f" {a=} {b=} {a.release_date=} {b.release_date=}"
+            )
+            raise ValueError(msg)
         with conn.cursor(name=f"emaildata_sync_{b.name}") as cursor:
             cursor.execute(
                 """

versions/admin.py

@@ -7,6 +7,7 @@
 from libraries.tasks import import_new_versions_tasks
 
 from . import models
+from .models import Version
 
 
 class VersionFileInline(admin.StackedInline):
@@ -23,9 +24,9 @@ class VersionAdmin(admin.ModelAdmin):
         "name",
         "release_date",
         "active",
-        "full_release",
         "beta",
         "fully_imported",
+        "full_release",
     ]
     list_filter = ["active", "full_release", "beta"]
     ordering = ["-release_date", "-name"]
@@ -34,6 +35,10 @@ class VersionAdmin(admin.ModelAdmin):
     inlines = [VersionFileInline]
     change_list_template = "admin/version_change_list.html"
 
+    def get_queryset(self, request: HttpRequest) -> QuerySet:
+        # we want all versions here, including not fully_imported
+        return Version.objects.with_partials()
+
     def get_urls(self):
         urls = super().get_urls()
         my_urls = [

versions/management/commands/import_archives_release_data.py

@@ -43,11 +43,11 @@ def command(release: str, new: bool):
     last_release = settings.MIN_ARCHIVES_RELEASE
 
     if release:
-        versions = Version.objects.filter(name__icontains=release)
+        versions = Version.objects.with_partials().filter(name__icontains=release)
     elif new:
-        versions = [Version.objects.most_recent()]
+        versions = [Version.objects.with_partials().most_recent()]
     else:
-        versions = Version.objects.filter(name__gte=last_release)
+        versions = Version.objects.with_partials().filter(name__gte=last_release)
     logger.info(f"import_archive_release_data {versions=}")
 
     for v in versions:

versions/managers.py

@@ -12,7 +12,7 @@
 class VersionQuerySet(models.QuerySet):
     def active(self):
         """Return active versions"""
-        return self.filter(active=True, fully_imported=True)
+        return self.filter(active=True)
 
     def most_recent(self):
         """Return most recent active non-beta version"""
@@ -61,6 +61,10 @@ def with_version_split(self):
 
 class VersionManager(models.Manager):
     def get_queryset(self):
+        return VersionQuerySet(self.model, using=self._db).filter(fully_imported=True)
+
+    def with_partials(self):
+        """Return all objects regardless of fully_imported status"""
         return VersionQuerySet(self.model, using=self._db)
 
     def active(self):

versions/releases.py

@@ -36,9 +36,7 @@ def get_download_uris_for_release(
         resp = session.get(release_path)
         resp.raise_for_status()
     except requests.exceptions.HTTPError as e:
-        logger.error(
-            "get_archives_releases_list_error", exc_msg=str(e), url=release_path
-        )
+        logger.error(f"get_archives_releases_list_error {str(e)=}, {release_path=}")
         raise
 
     soup = BeautifulSoup(resp.text, "html.parser")
@@ -207,7 +205,7 @@ def get_release_notes_for_version_s3(version_pk):
     #  and are not extensible if we encounter additional filename patterns in the
     #  future; we should refactor.
     try:
-        version = Version.objects.get(pk=version_pk)
+        version = Version.objects.with_partials().get(pk=version_pk)
     except Version.DoesNotExist:
         logger.info(
             "get_release_notes_for_version_s3_error_version_not_found",
@@ -244,7 +242,7 @@ def get_release_notes_for_version_github(version_pk):
     #  and are not extensible if we encounter additional filename patterns in the
     #  future; we should refactor.
     try:
-        version = Version.objects.get(pk=version_pk)
+        version = Version.objects.with_partials().get(pk=version_pk)
     except Version.DoesNotExist:
         logger.info(
             "get_release_notes_for_version_error_version_not_found",
@@ -319,7 +317,7 @@ def store_release_notes_for_version(version_pk):
     # Get the version
     # todo: convert to task, remove the task that calls this, is redundant
     try:
-        version = Version.objects.get(pk=version_pk)
+        version = Version.objects.with_partials().get(pk=version_pk)
     except Version.DoesNotExist:
         logger.info(f"store_release_notes version_not_found {version_pk=}")
         raise Version.DoesNotExist

versions/tasks.py

@@ -47,11 +47,11 @@ def import_versions(
             imports are finished.
     """
     if delete_versions:
-        Version.objects.all().delete()
+        Version.objects.with_partials().all().delete()
         logger.info("import_versions_deleted_all_versions")
 
     # delete any versions that were only partially imported so they are re-imported
-    Version.objects.filter(fully_imported=False).delete()
+    Version.objects.with_partials().filter(fully_imported=False).delete()
 
     # Get all Boost tags from Github
     client = GithubAPIClient(token=token)
@@ -82,7 +82,7 @@ def import_versions(
 def import_release_notes(new_versions_only=True):
     """Imports release notes from the existing rendered
     release notes in the repository."""
-    versions = [Version.objects.most_recent()]
+    versions = [Version.objects.with_partials().most_recent()]
     if not new_versions_only:
         versions = Version.objects.exclude(name__in=["master", "develop"]).active()
 
@@ -96,7 +96,7 @@ def import_release_notes(new_versions_only=True):
 def store_release_notes_task(version_pk):
     """Stores the release notes for a single version."""
     try:
-        Version.objects.get(pk=version_pk)
+        Version.objects.with_partials().get(pk=version_pk)
     except Version.DoesNotExist:
         logger.error(f"store_release_notes_task_version_does_not_exist {version_pk=}")
         return
@@ -132,7 +132,7 @@ def import_version(
     else:
         data = {}
 
-    version, created = Version.objects.update_or_create(
+    version, created = Version.objects.with_partials().update_or_create(
         name=name,
         defaults={
             "github_url": f"{base_url}{name}",
@@ -161,17 +161,29 @@ def import_development_versions():
     """Imports the `master` and `develop` branches as Versions"""
     base_url = "https://github.com/boostorg/boost/tree/"
 
+    import_version_tasks = []
+    import_library_version_tasks = []
     for branch in settings.BOOST_BRANCHES:
-        import_version.delay(
-            branch,
-            branch,
-            beta=False,
-            full_release=False,
-            get_release_date=False,
-            base_url=base_url,
+        import_version_tasks.append(
+            import_version.s(
+                branch,
+                branch,
+                beta=False,
+                full_release=False,
+                get_release_date=False,
+                base_url=base_url,
+            )
         )
 
-        import_library_versions.delay(branch, version_type="branch")
+        import_library_version_tasks.append(
+            import_library_versions.s(branch, version_type="branch")
+        )
+    # download all versions before library version data, only then mark fully completed
+    version_group = group(*import_version_tasks)
+    library_group = group(*import_library_version_tasks)
+    version_group.link(library_group)
+    library_group.link(mark_fully_completed.s())
+    version_group()
 
 
 @app.task
@@ -264,7 +276,7 @@ def import_library_versions(version_name, token=None, version_type="tag"):
     """For a specific version, imports all LibraryVersions using GitHub data"""
     # todo: this needs to be refactored and tests added
     try:
-        version = Version.objects.get(name=version_name)
+        version = Version.objects.with_partials().get(name=version_name)
     except Version.DoesNotExist:
         logger.info(
             "import_library_versions_version_not_found", version_name=version_name
@@ -401,7 +413,7 @@ def import_library_versions(version_name, token=None, version_type="tag"):
 @app.task
 def import_release_downloads(version_pk):
     logger.info(f"import_release_downloads w/ {version_pk=}")
-    version = Version.objects.get(pk=version_pk)
+    version = Version.objects.with_partials().get(pk=version_pk)
     version_num = version.name.replace("boost-", "")
     if version_num < "1.63.0":
         # Downloads are in Sourceforge for older versions, and that has
@@ -422,11 +434,9 @@ def get_release_date_for_version(version_pk, commit_sha, token=None):
     :param commit_sha: The SHA of the commit to get the release date for.
     """
     try:
-        version = Version.objects.get(pk=version_pk)
+        version = Version.objects.with_partials().get(pk=version_pk)
     except Version.DoesNotExist:
-        logger.error(
-            "get_release_date_for_version_no_version_found", version_pk=version_pk
-        )
+        logger.error(f"get_release_date_for_version_no_version_found {version_pk=}")
         return
 
     if not token:
@@ -485,7 +495,9 @@ def purge_fastly_release_cache():
 @app.task
 def mark_fully_completed():
     """Marks all versions as fully imported"""
-    Version.objects.filter(fully_imported=False).update(fully_imported=True)
+    Version.objects.with_partials().filter(fully_imported=False).update(
+        fully_imported=True
+    )
     logger.info("Marked all versions as fully imported.")
 
 
@@ -510,7 +522,7 @@ def skip_tag(name, new=False):
     EXCLUSIONS = ["beta", "-rc", "-bgl"]
 
     # If we are only importing new versions, and we already have this one, skip
-    if new and Version.objects.filter(name=name).exists():
+    if new and Version.objects.with_partials().filter(name=name).exists():
         return True
 
     # If this version falls in our exclusion list, skip it