1
0
Fork 0
mirror of https://github.com/imjasonh/urls synced 2026-07-17 14:43:06 +00:00
urls/crawl.sh
2023-01-26 11:28:34 -05:00

19 lines
456 B
Bash
Executable file

#! /usr/bin/env bash
set -euo pipefail
while read l; do
[[ -z $l ]] && continue
[[ $l =~ ^#.* ]] && continue
fn=${l#"https://"}
echo $fn
mkdir -p $(dirname $fn)
curl -s $l | jq > $fn
if [ "$(jq 'has("keys")' $fn)" == "true" ]; then
jq -S -s 'sort_by(.keys[].kid)' $fn > ${TMPDIR}/out && mv ${TMPDIR}/out $fn
fi
if grep -q "BEGIN CERTIFICATE" $fn; then
jq -S '.' $fn > ${TMPDIR}/out && mv ${TMPDIR}/out $fn
fi
done < urls.txt