Julien Floret: 1 release: do not publish internal artifact formats 12 files changed, 45 insertions(+), 6 deletions(-)
Copy & paste the following snippet into your terminal to import this patchset into git:
curl -s https://lists.sr.ht/~rjarry/dlrepo/patches/44106/mbox | git am -3Learn more about email & git
Some artifact formats can be for internal use only and never meant to be published (e.g. test reports, preconfigured VM snapshots...). It is tedious and error-prone to have to systematically exclude them from the customer ACLs on the public server. Furthermore, it can be a waste of storage. Reintroduce the notion of "internal" formats; these formats are skipped when publishing a job. Internal formats are filtered via the "DLREPO_INTERNAL_FORMAT_FILTER" config option: all format names that match this regular expression are considered internal. They are lined in red in the web UI to be easily recognizable. Note: these formats are not taken into account when calculating the job digest, because this digest is used to compare private and public jobs. Signed-off-by: Julien Floret <julien.floret@6wind.com> Acked-by: Thomas Faivre <thomas.faivre@6wind.com>
Acked-by: Robin-Jarry <robin@jarry.cc> Applied. Thanks!
--- dlrepo/fs/fmt.py | 11 +++++++++++ dlrepo/fs/job.py | 11 ++++++++--- dlrepo/fs/tag.py | 2 +- dlrepo/templates/job.html | 5 ++++- dlrepo/templates/product_version.html | 5 ++++- dlrepo/views/fmt.py | 1 + dlrepo/views/job.py | 1 + dlrepo/views/product.py | 1 + dlrepo/views/util.py | 1 + docs/dlrepo-config.5.scdoc | 6 ++++++ etc/default | 1 + scss/sections.scss | 6 ++++++ 12 files changed, 45 insertions(+), 6 deletions(-) diff --git a/dlrepo/fs/fmt.py b/dlrepo/fs/fmt.py index d003383b2e98..487bff1d4c7e 100644 --- a/dlrepo/fs/fmt.py +++ b/dlrepo/fs/fmt.py @@ -17,6 +17,12 @@ from .util import SubDir, file_digest LOG = logging.getLogger(__name__) +INTERNAL_FORMAT_FILTER = os.getenv("DLREPO_INTERNAL_FORMAT_FILTER") +if INTERNAL_FORMAT_FILTER: + INTERNAL_FORMAT_RE = re.compile(INTERNAL_FORMAT_FILTER) +else: + INTERNAL_FORMAT_RE = None + # -------------------------------------------------------------------------------------- class ArtifactFormat(SubDir): @@ -27,6 +33,11 @@ class ArtifactFormat(SubDir): def url_bit(self) -> str: return self.name + def is_internal(self) -> bool: + if INTERNAL_FORMAT_RE is None: + return False + return INTERNAL_FORMAT_RE.match(self.name) is not None + def get_files(self) -> Iterator[str]: for root, dirs, files in os.walk(self._path): dirs.sort() diff --git a/dlrepo/fs/job.py b/dlrepo/fs/job.py index cc177ba15ffb..662f52dde964 100644 --- a/dlrepo/fs/job.py +++ b/dlrepo/fs/job.py @@ -45,8 +45,13 @@ class Job(SubDir): def timestamp(self) -> int: return Job.creation_date(self) - def get_formats(self) -> Iterator[ArtifactFormat]: - yield from ArtifactFormat.all(self) + def get_formats(self, exclude_internal: bool = False) -> Iterator[ArtifactFormat]: + if exclude_internal: + for fmt in ArtifactFormat.all(self): + if not fmt.is_internal(): + yield fmt + else: + yield from ArtifactFormat.all(self) def get_format(self, name: str) -> ArtifactFormat: return ArtifactFormat(self, name) @@ -166,7 +171,7 @@ class Job(SubDir): async def set_digest(self): loop = asyncio.get_running_loop() digests = {} - for fmt in self.get_formats(): + for fmt in self.get_formats(exclude_internal=True): if fmt.name == "container": _, digest = self.root().container_registry.manifest_by_parent( self._path diff --git a/dlrepo/fs/tag.py b/dlrepo/fs/tag.py index 1378f7549d15..35878efcad20 100644 --- a/dlrepo/fs/tag.py +++ b/dlrepo/fs/tag.py @@ -199,7 +199,7 @@ class Tag(SubDir): job.set_released(False) self._publish_status_path().write_text(f"uploading {job.name}\n") tasks = [] - for fmt in job.get_formats(): + for fmt in job.get_formats(exclude_internal=True): tasks.append(loop.create_task(self._publish_fmt(fmt, sess, semaphore))) await asyncio.gather(*tasks) metadata = job.get_metadata() diff --git a/dlrepo/templates/job.html b/dlrepo/templates/job.html index 6be0ee66b9cb..d9e43a4068c5 100644 --- a/dlrepo/templates/job.html +++ b/dlrepo/templates/job.html @@ -18,7 +18,10 @@ <hr/> <div class="artifact-formats"> {% for fmt in formats|sort(attribute="name") %} - <div class="artifact-format"> + <div class="artifact-format{% if fmt.internal %} internal{% endif %}" + {% if fmt.internal %} + title="internal format (not for release)" + {% endif %}> <a href="{{fmt.name}}" class="format">{{fmt.name}}/</a> <a href="{{fmt.name}}.tar" class="archive" title="Whole archive"> {{fmt.name}}.tar diff --git a/dlrepo/templates/product_version.html b/dlrepo/templates/product_version.html index 9dc785181acd..65ecc868cc4b 100644 --- a/dlrepo/templates/product_version.html +++ b/dlrepo/templates/product_version.html @@ -14,7 +14,10 @@ <hr/> <div class="artifact-formats"> {% for fmt in version.artifact_formats|sort(attribute="name") %} - <div class="artifact-format"> + <div class="artifact-format{% if fmt.internal %} internal{% endif %}" + {% if fmt.internal %} + title="internal format (not for release)" + {% endif %}> <a href="{{fmt.name}}" class="format">{{fmt.name}}/</a> <a href="{{fmt.name}}.tar" class="archive" title="Whole archive"> {{fmt.name}}.tar diff --git a/dlrepo/views/fmt.py b/dlrepo/views/fmt.py index 05f92b2bae5e..5378cb9e8016 100644 --- a/dlrepo/views/fmt.py +++ b/dlrepo/views/fmt.py @@ -42,6 +42,7 @@ class FormatDirView(BaseView): data = { "artifact_format": { "name": fmt.name, + "internal": fmt.is_internal(), "dirty": fmt.is_dirty(), "files": list(fmt.get_digests().keys()), }, diff --git a/dlrepo/views/job.py b/dlrepo/views/job.py index 644cd3ae3576..eb1d9f9de58f 100644 --- a/dlrepo/views/job.py +++ b/dlrepo/views/job.py @@ -73,6 +73,7 @@ class JobView(BaseView): formats.append( { "name": f.name, + "internal": f.is_internal(), "rpm": rpm, "deb": deb, "url": fmt_url, diff --git a/dlrepo/views/product.py b/dlrepo/views/product.py index 5d8b3d55ec8e..c54709db6288 100644 --- a/dlrepo/views/product.py +++ b/dlrepo/views/product.py @@ -196,6 +196,7 @@ class VersionView(BaseView): formats.append( { "name": fmt.name, + "internal": fmt.is_internal(), "rpm": rpm, "deb": deb, "url": fmt_url, diff --git a/dlrepo/views/util.py b/dlrepo/views/util.py index c231b0272473..97ebc367bda2 100644 --- a/dlrepo/views/util.py +++ b/dlrepo/views/util.py @@ -67,6 +67,7 @@ class BaseView(web.View): "artifact_format": { "name": fmt.name, "relpath": relpath.rstrip("/"), + "internal": fmt.is_internal(), "dirs": dirs, "files": files, }, diff --git a/docs/dlrepo-config.5.scdoc b/docs/dlrepo-config.5.scdoc index 4e50e811c1f5..ec873da46cf6 100644 --- a/docs/dlrepo-config.5.scdoc +++ b/docs/dlrepo-config.5.scdoc @@ -274,6 +274,12 @@ running the daemon process. Maximum number of concurrent requests made to *DLREPO_PUBLISH_URL* when publishing tags. +*DLREPO_INTERNAL_FORMAT_FILTER* (optional) + Regular expression for selecting which artifact formats must not be + published. If not set, all formats are published. + + Example: _'^debug$'_ + # SEE ALSO *dlrepo*(7), diff --git a/etc/default b/etc/default index 2d357c8d8d41..b99a3a278861 100644 --- a/etc/default +++ b/etc/default @@ -10,6 +10,7 @@ #DLREPO_USER_QUOTA=10737418240 #DLREPO_POST_PROCESS_CMD= #DLREPO_POST_PROCESS_FILTER= +#DLREPO_INTERNAL_FORMAT_FILTER= #DLREPO_PUBLIC_URL= #DLREPO_TEMPLATES_DIR= #DLREPO_STATIC_DIR= diff --git a/scss/sections.scss b/scss/sections.scss index e8872791dbe6..a3f94e488f83 100644 --- a/scss/sections.scss +++ b/scss/sections.scss @@ -104,6 +104,12 @@ section.files { > .artifact-format:hover { background-color: lighten($lightgray, 5%); } + > .artifact-format.internal { + border-color: #fca5a5; + } + > .artifact-format.internal:hover { + background-color: #fee2e2; + } } > .whole-archive { -- 2.39.2
Julien Floret, Aug 29, 2023 at 10:26:
builds.sr.ht <builds@sr.ht>dlrepo/patches/.build.yml: SUCCESS in 2m12s [release: do not publish internal artifact formats][0] from [Julien Floret][1] [0]: https://lists.sr.ht/~rjarry/dlrepo/patches/44106 [1]: mailto:julien.floret@6wind.com ✓ #1049355 SUCCESS dlrepo/patches/.build.yml https://builds.sr.ht/~rjarry/job/1049355