summaryrefslogtreecommitdiff
path: root/Makefile
diff options
context:
space:
mode:
Diffstat (limited to 'Makefile')
-rw-r--r--Makefile29
1 files changed, 17 insertions, 12 deletions
diff --git a/Makefile b/Makefile
index 14db651..eb8ae8e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,24 +1,29 @@
+PATH:=$(CURDIR)/bin:$(PATH)
+export PATH
+
all: each-cdx
-ftp.json: cdx_json
- ./cdx_json 'url=ftp.unicode.org/Public/*' 'collapse=urlkey' > $@
-www.json: cdx_json
- ./cdx_json 'url=www.unicode.org/Public/*' 'collapse=urlkey' > $@
-urlkeys.txt: ftp.json www.json Makefile
- cat $^ | cut -d '"' -f2,10 | sed -n 's/"200$$//p' | cut -d '?' -f1 | sed 's/,ftp)/)/' | sort -u | grep -i -e cvtutf -e convertutf > $@
-urlkeys.mk: urlkeys.txt
- sed 's/^/urlkeys+=/' < $< > $@
+dat:
+ mkdir -p $@
+dat/ftp.txt: | dat
+ cdxget 'url=ftp.unicode.org/Public/*' 'fl=urlkey' 'filter=statuscode:200' 'filter=urlkey:.*(cvt|convert)utf.*' > $@
+dat/www.txt: | dat
+ cdxget 'url=www.unicode.org/Public/*' 'fl=urlkey' 'filter=statuscode:200' 'filter=urlkey:.*(cvt|convert)utf.*' > $@
+dat/urlkeys.txt: dat/ftp.txt dat/www.txt
+ cat $^ | cut -d '?' -f1 | sed 's/,ftp)/)/' | sort -u > $@
+dat/urlkeys.mk: dat/urlkeys.txt
+ cat $^ | sed 's/^/urlkeys+=/' < $< > $@
--include urlkeys.mk
+-include dat/urlkeys.mk
rp = )
c = ,
all_urlkeys = $(urlkeys) $(subst $(rp),$(c)ftp$(rp),$(urlkeys))
-each-cdx/%.json: cdx_json urlkey2url
+dat/each-cdx/%.txt:
mkdir -p '$(@D)'
- ./cdx_json "url=$$(./urlkey2url '$*')" 'collapse=digest' > '$@'
+ cdxget "url=$$(urlkey2url '$*')" 'filter=statuscode:200' 'collapse=digest' 'fl=timestamp,original' > '$@'
-each-cdx: $(addprefix each-cdx/,$(addsuffix .json,$(all_urlkeys)))
+each-cdx: $(addprefix dat/each-cdx/,$(addsuffix .txt,$(all_urlkeys)))
.PHONY: each-cdx
.DELETE_ON_ERROR: