Some artifact formats can be for internal use only and never meant to
be published (e.g. test reports, preconfigured VM snapshots...). It is
tedious and error-prone to have to systematically exclude them from
the customer ACLs on the public server. Furthermore, it can be a waste
of storage.
Reintroduce the notion of "internal" formats; these formats are
skipped when publishing a job.
Internal formats are filtered via the "DLREPO_INTERNAL_FORMAT_FILTER"
config option: all format names that match this regular expression are
considered internal. They are lined in red in the web UI to be easily
recognizable.
Note: these formats are not taken into account when calculating
the job digest, because this digest is used to compare private and
public jobs.
Signed-off-by: Julien Floret <julien.floret@6wind.com>
Acked-by: Thomas Faivre <thomas.faivre@6wind.com>
---
dlrepo/fs/fmt.py | 11 +++++++++++
dlrepo/fs/job.py | 11 ++++++++---
dlrepo/fs/tag.py | 2 +-
dlrepo/templates/job.html | 5 ++++-
dlrepo/templates/product_version.html | 5 ++++-
dlrepo/views/fmt.py | 1 +
dlrepo/views/job.py | 1 +
dlrepo/views/product.py | 1 +
dlrepo/views/util.py | 1 +
docs/dlrepo-config.5.scdoc | 6 ++++++
etc/default | 1 +
scss/sections.scss | 6 ++++++
12 files changed, 45 insertions(+), 6 deletions(-)
diff --git a/dlrepo/fs/fmt.py b/dlrepo/fs/fmt.py
index d003383b2e98..487bff1d4c7e 100644
--- a/dlrepo/fs/fmt.py
+++ b/dlrepo/fs/fmt.py
@@ -17,6 +17,12 @@ from .util import SubDir, file_digest
LOG = logging.getLogger(__name__)
+INTERNAL_FORMAT_FILTER = os.getenv("DLREPO_INTERNAL_FORMAT_FILTER")
+if INTERNAL_FORMAT_FILTER:
+ INTERNAL_FORMAT_RE = re.compile(INTERNAL_FORMAT_FILTER)
+else:
+ INTERNAL_FORMAT_RE = None
+
# --------------------------------------------------------------------------------------
class ArtifactFormat(SubDir):
@@ -27,6 +33,11 @@ class ArtifactFormat(SubDir):
def url_bit(self) -> str:
return self.name
+ def is_internal(self) -> bool:
+ if INTERNAL_FORMAT_RE is None:
+ return False
+ return INTERNAL_FORMAT_RE.match(self.name) is not None
+
def get_files(self) -> Iterator[str]:
for root, dirs, files in os.walk(self._path):
dirs.sort()
diff --git a/dlrepo/fs/job.py b/dlrepo/fs/job.py
index cc177ba15ffb..662f52dde964 100644
--- a/dlrepo/fs/job.py
+++ b/dlrepo/fs/job.py
@@ -45,8 +45,13 @@ class Job(SubDir):
def timestamp(self) -> int:
return Job.creation_date(self)
- def get_formats(self) -> Iterator[ArtifactFormat]:
- yield from ArtifactFormat.all(self)
+ def get_formats(self, exclude_internal: bool = False) -> Iterator[ArtifactFormat]:
+ if exclude_internal:
+ for fmt in ArtifactFormat.all(self):
+ if not fmt.is_internal():
+ yield fmt
+ else:
+ yield from ArtifactFormat.all(self)
def get_format(self, name: str) -> ArtifactFormat:
return ArtifactFormat(self, name)
@@ -166,7 +171,7 @@ class Job(SubDir):
async def set_digest(self):
loop = asyncio.get_running_loop()
digests = {}
- for fmt in self.get_formats():
+ for fmt in self.get_formats(exclude_internal=True):
if fmt.name == "container":
_, digest = self.root().container_registry.manifest_by_parent(
self._path
diff --git a/dlrepo/fs/tag.py b/dlrepo/fs/tag.py
index 1378f7549d15..35878efcad20 100644
--- a/dlrepo/fs/tag.py
+++ b/dlrepo/fs/tag.py
@@ -199,7 +199,7 @@ class Tag(SubDir):
job.set_released(False)
self._publish_status_path().write_text(f"uploading {job.name}\n")
tasks = []
- for fmt in job.get_formats():
+ for fmt in job.get_formats(exclude_internal=True):
tasks.append(loop.create_task(self._publish_fmt(fmt, sess, semaphore)))
await asyncio.gather(*tasks)
metadata = job.get_metadata()
diff --git a/dlrepo/templates/job.html b/dlrepo/templates/job.html
index 6be0ee66b9cb..d9e43a4068c5 100644
--- a/dlrepo/templates/job.html
+++ b/dlrepo/templates/job.html
@@ -18,7 +18,10 @@
<hr/>
<div class="artifact-formats">
{% for fmt in formats|sort(attribute="name") %}
- <div class="artifact-format">
+ <div class="artifact-format{% if fmt.internal %} internal{% endif %}"
+ {% if fmt.internal %}
+ title="internal format (not for release)"
+ {% endif %}>
<a href="{{fmt.name}}" class="format">{{fmt.name}}/</a>
<a href="{{fmt.name}}.tar" class="archive" title="Whole archive">
{{fmt.name}}.tar
diff --git a/dlrepo/templates/product_version.html b/dlrepo/templates/product_version.html
index 9dc785181acd..65ecc868cc4b 100644
--- a/dlrepo/templates/product_version.html
+++ b/dlrepo/templates/product_version.html
@@ -14,7 +14,10 @@
<hr/>
<div class="artifact-formats">
{% for fmt in version.artifact_formats|sort(attribute="name") %}
- <div class="artifact-format">
+ <div class="artifact-format{% if fmt.internal %} internal{% endif %}"
+ {% if fmt.internal %}
+ title="internal format (not for release)"
+ {% endif %}>
<a href="{{fmt.name}}" class="format">{{fmt.name}}/</a>
<a href="{{fmt.name}}.tar" class="archive" title="Whole archive">
{{fmt.name}}.tar
diff --git a/dlrepo/views/fmt.py b/dlrepo/views/fmt.py
index 05f92b2bae5e..5378cb9e8016 100644
--- a/dlrepo/views/fmt.py
+++ b/dlrepo/views/fmt.py
@@ -42,6 +42,7 @@ class FormatDirView(BaseView):
data = {
"artifact_format": {
"name": fmt.name,
+ "internal": fmt.is_internal(),
"dirty": fmt.is_dirty(),
"files": list(fmt.get_digests().keys()),
},
diff --git a/dlrepo/views/job.py b/dlrepo/views/job.py
index 644cd3ae3576..eb1d9f9de58f 100644
--- a/dlrepo/views/job.py
+++ b/dlrepo/views/job.py
@@ -73,6 +73,7 @@ class JobView(BaseView):
formats.append(
{
"name": f.name,
+ "internal": f.is_internal(),
"rpm": rpm,
"deb": deb,
"url": fmt_url,
diff --git a/dlrepo/views/product.py b/dlrepo/views/product.py
index 5d8b3d55ec8e..c54709db6288 100644
--- a/dlrepo/views/product.py
+++ b/dlrepo/views/product.py
@@ -196,6 +196,7 @@ class VersionView(BaseView):
formats.append(
{
"name": fmt.name,
+ "internal": fmt.is_internal(),
"rpm": rpm,
"deb": deb,
"url": fmt_url,
diff --git a/dlrepo/views/util.py b/dlrepo/views/util.py
index c231b0272473..97ebc367bda2 100644
--- a/dlrepo/views/util.py
+++ b/dlrepo/views/util.py
@@ -67,6 +67,7 @@ class BaseView(web.View):
"artifact_format": {
"name": fmt.name,
"relpath": relpath.rstrip("/"),
+ "internal": fmt.is_internal(),
"dirs": dirs,
"files": files,
},
diff --git a/docs/dlrepo-config.5.scdoc b/docs/dlrepo-config.5.scdoc
index 4e50e811c1f5..ec873da46cf6 100644
--- a/docs/dlrepo-config.5.scdoc
+++ b/docs/dlrepo-config.5.scdoc
@@ -274,6 +274,12 @@ running the daemon process.
Maximum number of concurrent requests made to *DLREPO_PUBLISH_URL* when
publishing tags.
+*DLREPO_INTERNAL_FORMAT_FILTER* (optional)
+ Regular expression for selecting which artifact formats must not be
+ published. If not set, all formats are published.
+
+ Example: _'^debug$'_
+
# SEE ALSO
*dlrepo*(7),
diff --git a/etc/default b/etc/default
index 2d357c8d8d41..b99a3a278861 100644
--- a/etc/default
+++ b/etc/default
@@ -10,6 +10,7 @@
#DLREPO_USER_QUOTA=10737418240
#DLREPO_POST_PROCESS_CMD=
#DLREPO_POST_PROCESS_FILTER=
+#DLREPO_INTERNAL_FORMAT_FILTER=
#DLREPO_PUBLIC_URL=
#DLREPO_TEMPLATES_DIR=
#DLREPO_STATIC_DIR=
diff --git a/scss/sections.scss b/scss/sections.scss
index e8872791dbe6..a3f94e488f83 100644
--- a/scss/sections.scss
+++ b/scss/sections.scss
@@ -104,6 +104,12 @@ section.files {
> .artifact-format:hover {
background-color: lighten($lightgray, 5%);
}
+ > .artifact-format.internal {
+ border-color: #fca5a5;
+ }
+ > .artifact-format.internal:hover {
+ background-color: #fee2e2;
+ }
}
> .whole-archive {
--
2.39.2
Julien Floret, Aug 29, 2023 at 10:26:
> Some artifact formats can be for internal use only and never meant to
> be published (e.g. test reports, preconfigured VM snapshots...). It is
> tedious and error-prone to have to systematically exclude them from
> the customer ACLs on the public server. Furthermore, it can be a waste
> of storage.
>
> Reintroduce the notion of "internal" formats; these formats are
> skipped when publishing a job.
> Internal formats are filtered via the "DLREPO_INTERNAL_FORMAT_FILTER"
> config option: all format names that match this regular expression are
> considered internal. They are lined in red in the web UI to be easily
> recognizable.
>
> Note: these formats are not taken into account when calculating
> the job digest, because this digest is used to compare private and
> public jobs.
>
> Signed-off-by: Julien Floret <julien.floret@6wind.com>
> Acked-by: Thomas Faivre <thomas.faivre@6wind.com>
Acked-by: Robin-Jarry <robin@jarry.cc>
Applied. Thanks!