---
src/fead.py | 24 ++++++++++++++++++++----
1 file changed, 20 insertions(+), 4 deletions(-)
diff --git a/src/fead.py b/src/fead.py
index 370b623..0b35119 100755
--- a/src/fead.py
+++ b/src/fead.py
@@ -185,11 +185,23 @@ async def fetch(raw_url):
response.getheaders(), response)
-async def fetch_all(urls):
+async def fetch_skip_error(url):
+ try:
+ return await fetch(url)
+ except Exception as e:
+ warn(f'fail to fetch {url}: {e}',
+ type('FailureWarning', (Warning,), {}))
+ return None
+
+
+async def fetch_all(urls, skip_error):
"""Fetch all given URLs asynchronously and return them parsed."""
- tasks = gather(*map(fetch, urls))
+ if skip_error:
+ tasks = gather(*map(fetch_skip_error, urls))
+ else:
+ tasks = gather(*map(fetch, urls))
try:
- return await tasks
+ return filter(lambda t: t is not None, await tasks)
except:
tasks.cancel() # structured concurrency
raise
@@ -234,11 +246,15 @@ def main():
parser.add_argument('-o', '--output', metavar='PATH',
type=FileType('w'), default=stdout,
help='output file (default to stdout)')
+ parser.add_argument('-s', '--skip-error', action='store_true',
+ default=False,
+ help="errors not causing failure but logged")
args = parser.parse_args()
template = args.template.read()
args.template.close()
- for ad in select(args.count, (ad for feed in run(fetch_all(args.feeds))
+ for ad in select(args.count, (ad for feed in run(fetch_all(args.feeds,
+ args.skip_error))
for ad in select(args.per_feed, feed))):
args.output.write(template.format(**truncate(ad, args.len)._asdict()))
args.output.close()
--
2.40.1