Skip to content

web search done right

New strategy for git-lfs structure. RocksDb changes it's index each time it is opened which leads to many uploads to lfs. Also for some reason, the index in lfs seems to be broken.

Mikkel Denkere313b1c6cc95
diff --git a/.gitattributes b/.gitattributes
index 20739a9d..f6bd2897 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1 +1 @@
-data/** filter=lfs diff=lfs merge=lfs -text
+data.tar.gz filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
index ba941e4a..1ae26420 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,15 +1,9 @@
/target
.DS_Store
flamegraph.svg
-/data/enwiki-20220801-pages-articles-multistream.xml.bz2
-/data/warc_files/CC-MAIN-20220116093137-20220116123137-00000.warc.gz
-/data/warc_files/CC-MAIN-20220116093137-20220116123137-00001.warc.gz
-/data/warc_files/CC-MAIN-20220116093137-20220116123137-00003.warc.gz
static/**/*.css
node_modules
package-lock.json
notes
*.log
-/data/**/LOG.old*
-/data/webgraph
-/data/warc_files
+/data/**/*
diff --git a/data.tar.gz b/data.tar.gz
new file mode 100644
index 00000000..970d26d3
--- /dev/null
+++ b/data.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cabcd57754add6d968fb719d9c4e8a6d259a758995163027f7714ffec1e35d4
+size 941862585
diff --git a/data/.gitkeep b/data/.gitkeep
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/centrality/conf b/data/centrality/conf
deleted file mode 100755
index 89002549..00000000
--- a/data/centrality/conf
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:88a3aa53a63fd1728340d571e01a7983c9deb39be1b285b59ea6fad3adc052d8
-size 61
diff --git a/data/centrality/db b/data/centrality/db
deleted file mode 100755
index 6138769b..00000000
--- a/data/centrality/db
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f56e6255edea4fa061f88789c5b5b1b0a26b0d0b3ebb7f5955a5c10276ca329e
-size 42467328
diff --git a/data/centrality/snap.0000000009100BE4 b/data/centrality/snap.0000000009100BE4
deleted file mode 100755
index 36c3e9a8..00000000
--- a/data/centrality/snap.0000000009100BE4
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2063ab43eb74a396cbd2ad6b67c00c32230b580a29c37032e0de1c5fd6cf9d00
-size 218377
diff --git a/data/entity/attribute_occurrences/000009.sst b/data/entity/attribute_occurrences/000009.sst
deleted file mode 100644
index 0267052c..00000000
--- a/data/entity/attribute_occurrences/000009.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4f56c95c5d31f8802c40f599ba4508fb6ef0ffffe6baf5b3e144cce938e4c3bb
-size 174847
diff --git a/data/entity/attribute_occurrences/000011.sst b/data/entity/attribute_occurrences/000011.sst
deleted file mode 100644
index ba0a86f0..00000000
--- a/data/entity/attribute_occurrences/000011.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:65270d2bfbe4aa90f7b5f583dd6755c9a47418a85de33c9eeda11b94e439b1b9
-size 226925
diff --git a/data/entity/attribute_occurrences/000013.sst b/data/entity/attribute_occurrences/000013.sst
deleted file mode 100644
index 6edb961e..00000000
--- a/data/entity/attribute_occurrences/000013.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:83717d6b717dbc2254b7c09130d888f898851e3eec62e1a541aed78da9d24b06
-size 146557
diff --git a/data/entity/attribute_occurrences/CURRENT b/data/entity/attribute_occurrences/CURRENT
deleted file mode 100644
index 194192bf..00000000
--- a/data/entity/attribute_occurrences/CURRENT
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0c27ed6e0a0e47b52cf24da720967bfa648a86af64dbaed7f0248a9d0cc94b66
-size 16
diff --git a/data/entity/attribute_occurrences/IDENTITY b/data/entity/attribute_occurrences/IDENTITY
deleted file mode 100644
index 72599f90..00000000
--- a/data/entity/attribute_occurrences/IDENTITY
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:138fd3d1d88f180b677bbfc64d80d3c4d96de802d58da72f5f7ac0856a623d71
-size 36
diff --git a/data/entity/attribute_occurrences/LOCK b/data/entity/attribute_occurrences/LOCK
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/entity/attribute_occurrences/LOG b/data/entity/attribute_occurrences/LOG
deleted file mode 100644
index 846d9204..00000000
--- a/data/entity/attribute_occurrences/LOG
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fb02808941c8a51f3f76379e1db7e89dbcddd9266ffbcb2ce65f939269fc7f86
-size 22983
diff --git a/data/entity/attribute_occurrences/MANIFEST-000057 b/data/entity/attribute_occurrences/MANIFEST-000057
deleted file mode 100644
index 18d28e27..00000000
--- a/data/entity/attribute_occurrences/MANIFEST-000057
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fb818c54efcaf96a16becbe4eed72e4bda32e4f81fb57cb1b5426afe6374ff8b
-size 428
diff --git a/data/entity/attribute_occurrences/MANIFEST-000132 b/data/entity/attribute_occurrences/MANIFEST-000132
deleted file mode 100644
index edc6424e..00000000
--- a/data/entity/attribute_occurrences/MANIFEST-000132
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0eb8f9fb6aa535b0d3e68cc1e8648334cc04f65ddacab57371416c87ef64d270
-size 430
diff --git a/data/entity/attribute_occurrences/OPTIONS-000055 b/data/entity/attribute_occurrences/OPTIONS-000055
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/entity/attribute_occurrences/OPTIONS-000055
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/entity/attribute_occurrences/OPTIONS-000060 b/data/entity/attribute_occurrences/OPTIONS-000060
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/entity/attribute_occurrences/OPTIONS-000060
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/entity/attribute_occurrences/OPTIONS-000130 b/data/entity/attribute_occurrences/OPTIONS-000130
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/entity/attribute_occurrences/OPTIONS-000130
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/entity/images/000016.sst b/data/entity/images/000016.sst
deleted file mode 100644
index f98d586a..00000000
--- a/data/entity/images/000016.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:eff6efc0f5d63961e6c67b364bcc00cb9124cf51a889a68560bed3f7d6b0d0d8
-size 67218210
diff --git a/data/entity/images/000017.sst b/data/entity/images/000017.sst
deleted file mode 100644
index 5b6e39ec..00000000
--- a/data/entity/images/000017.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1f3d794beb951cf3ef2bf560c723e9d354dfffc36cb5499db9e95bdb905d2b73
-size 67310472
diff --git a/data/entity/images/000018.sst b/data/entity/images/000018.sst
deleted file mode 100644
index 0814d4b4..00000000
--- a/data/entity/images/000018.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a74f57be3610ce8b4c223daf1063614d4e576b7001e0c6dad2bf8d598e36e622
-size 67213493
diff --git a/data/entity/images/000019.sst b/data/entity/images/000019.sst
deleted file mode 100644
index a3bf0d68..00000000
--- a/data/entity/images/000019.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ab1d15dd82ac67c1c8ab3adf097bd06f81ea5207342029ebcc73d6245c379abc
-size 60042860
diff --git a/data/entity/images/000021.sst b/data/entity/images/000021.sst
deleted file mode 100644
index 9cbe93b0..00000000
--- a/data/entity/images/000021.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9a151c8f0d006c1223b76aebc14120212c309daae346b6f0a08d1547ffec7077
-size 65661368
diff --git a/data/entity/images/000023.sst b/data/entity/images/000023.sst
deleted file mode 100644
index 7fb7f067..00000000
--- a/data/entity/images/000023.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d480a9623fcb9a4e9115a94bc8643548bd6ec659b395d93dc23edba8d57572f6
-size 61004478
diff --git a/data/entity/images/CURRENT b/data/entity/images/CURRENT
deleted file mode 100644
index 4c549b93..00000000
--- a/data/entity/images/CURRENT
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:eae9fea6aa6c724238caf1cd3b0fbd967fa9f57afeccc437e3c0e5d563891ee7
-size 16
diff --git a/data/entity/images/IDENTITY b/data/entity/images/IDENTITY
deleted file mode 100644
index 1e467612..00000000
--- a/data/entity/images/IDENTITY
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1689b459ee8a2aea971605f2f99f4742cda84f08e248afe84161e6e6c9500b12
-size 36
diff --git a/data/entity/images/LOCK b/data/entity/images/LOCK
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/entity/images/LOG b/data/entity/images/LOG
deleted file mode 100644
index da9cecd0..00000000
--- a/data/entity/images/LOG
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8cddc9638f2d59767edba2e54372b73581dc06690f9f4e7fd9e5d4e86a23bcdf
-size 23176
diff --git a/data/entity/images/MANIFEST-000067 b/data/entity/images/MANIFEST-000067
deleted file mode 100644
index d99b90ee..00000000
--- a/data/entity/images/MANIFEST-000067
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e0429f925c46995a24c552aa3695b00a41634a38fc7e5d4b7ec0f779b7bd5062
-size 775
diff --git a/data/entity/images/MANIFEST-000144 b/data/entity/images/MANIFEST-000144
deleted file mode 100644
index 43a370c6..00000000
--- a/data/entity/images/MANIFEST-000144
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:012a5cc96e37a41a51166cbf5bdc2c0e0995637094efef61df6a2d27ac7ca2ac
-size 910
diff --git a/data/entity/images/OPTIONS-000065 b/data/entity/images/OPTIONS-000065
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/entity/images/OPTIONS-000065
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/entity/images/OPTIONS-000070 b/data/entity/images/OPTIONS-000070
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/entity/images/OPTIONS-000070
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/entity/images/OPTIONS-000142 b/data/entity/images/OPTIONS-000142
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/entity/images/OPTIONS-000142
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/entity/inverted_index/.managed.json b/data/entity/inverted_index/.managed.json
deleted file mode 100644
index 9b1db3ae..00000000
--- a/data/entity/inverted_index/.managed.json
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3601da667cad6bb25d5ce6fe19c37d510e815b8c18ffe795e86b94f377035930
-size 258
diff --git a/data/entity/inverted_index/.tantivy-meta.lock b/data/entity/inverted_index/.tantivy-meta.lock
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/entity/inverted_index/.tantivy-writer.lock b/data/entity/inverted_index/.tantivy-writer.lock
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fast b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fast
deleted file mode 100644
index 7f089e2b..00000000
--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fast
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b253ab4389b9114e0d15974570866464bccebfcc29ba6a1f79f3808dbef25443
-size 99
diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fieldnorm b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fieldnorm
deleted file mode 100644
index d6a5a497..00000000
--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fieldnorm
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8b75ed3d24a4a59938e97561c1c2f5613ebeb4fa806022ae1e85708a56640be2
-size 600121
diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.idx b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.idx
deleted file mode 100644
index d2a593f8..00000000
--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.idx
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c27203416473e02b21a4b1e38135108d2c8b82a1a83dd3dfd4484afb0a50d388
-size 26057756
diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.pos b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.pos
deleted file mode 100644
index cd63e56a..00000000
--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.pos
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7d5c550f6cfeb8508ddbde9aa741a1ab225ba1595c36a364b24291dbaa665ad2
-size 20506426
diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.store b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.store
deleted file mode 100644
index e5c2c1db..00000000
--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.store
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f45131394b38d9c917f8b725c98ec80ae1967c60dd1f027b06bc0fda647476c1
-size 237946448
diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.term b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.term
deleted file mode 100644
index ca3ea7bd..00000000
--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.term
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c85d71c7961a3ae4c559661e4920c2bfd0cc97845f4b416f76765034bbeafc0b
-size 5387812
diff --git a/data/entity/inverted_index/meta.json b/data/entity/inverted_index/meta.json
deleted file mode 100644
index 62a38409..00000000
--- a/data/entity/inverted_index/meta.json
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7ae6733e1cb2efe06d6aeb3aea06d1face0ffa32cf835efaf770d91765009763
-size 1437
diff --git a/data/index/favicon_store/000009.sst b/data/index/favicon_store/000009.sst
deleted file mode 100644
index ffdacd79..00000000
--- a/data/index/favicon_store/000009.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bff70c998a5f6398310c53d1427156c19a0536c99f29696087171182ef991d5d
-size 130482
diff --git a/data/index/favicon_store/CURRENT b/data/index/favicon_store/CURRENT
deleted file mode 100644
index 0e9ee32d..00000000
--- a/data/index/favicon_store/CURRENT
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9c20c7d5e29035f91bef5b253fedeae4b6e37dea976d050b34e1819eca981750
-size 16
diff --git a/data/index/favicon_store/IDENTITY b/data/index/favicon_store/IDENTITY
deleted file mode 100644
index d9615944..00000000
--- a/data/index/favicon_store/IDENTITY
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e62e2a4e1c3c3514f88e78a47b831d082897ea7bf3dd1d7cd64b292431e3eb3a
-size 36
diff --git a/data/index/favicon_store/LOCK b/data/index/favicon_store/LOCK
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/index/favicon_store/LOG b/data/index/favicon_store/LOG
deleted file mode 100644
index cc01fb57..00000000
--- a/data/index/favicon_store/LOG
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e003c3f153939ed9ed29025b30af57059f20629d3c5f937397eaae3bb759263a
-size 19788
diff --git a/data/index/favicon_store/MANIFEST-000058 b/data/index/favicon_store/MANIFEST-000058
deleted file mode 100644
index f1783b02..00000000
--- a/data/index/favicon_store/MANIFEST-000058
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:409accc944bd2c8a32b7796a43192dc10e1b7005580497a65c1315a218a6a3e1
-size 173
diff --git a/data/index/favicon_store/MANIFEST-000158 b/data/index/favicon_store/MANIFEST-000158
deleted file mode 100644
index 1f7a0b8a..00000000
--- a/data/index/favicon_store/MANIFEST-000158
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:28ab7a432c135fabb46de3ea420fb8d21d5abb9a47e5879748bfffd9f9f115f0
-size 176
diff --git a/data/index/favicon_store/OPTIONS-000056 b/data/index/favicon_store/OPTIONS-000056
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/index/favicon_store/OPTIONS-000056
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/favicon_store/OPTIONS-000061 b/data/index/favicon_store/OPTIONS-000061
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/index/favicon_store/OPTIONS-000061
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/favicon_store/OPTIONS-000156 b/data/index/favicon_store/OPTIONS-000156
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/index/favicon_store/OPTIONS-000156
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/inverted_index/.managed.json b/data/index/inverted_index/.managed.json
deleted file mode 100644
index 6cc3f9cf..00000000
--- a/data/index/inverted_index/.managed.json
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8255799db4e3bd16e19fd14bdf9c6612b795f0853aa4ca06634088e2cf8927d6
-size 258
diff --git a/data/index/inverted_index/.tantivy-meta.lock b/data/index/inverted_index/.tantivy-meta.lock
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/index/inverted_index/.tantivy-writer.lock b/data/index/inverted_index/.tantivy-writer.lock
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fast b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fast
deleted file mode 100644
index 80cbe8b7..00000000
--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fast
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a15c7a425ed19862955c091f08ca6f2f72c06d52d9c4c6e659b807ee12557159
-size 229028
diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fieldnorm b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fieldnorm
deleted file mode 100644
index c875d09c..00000000
--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fieldnorm
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7c084af14222dba37866f2a51700306bb324efc3243acfa43112544155a506a1
-size 318675
diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.idx b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.idx
deleted file mode 100644
index bba4358a..00000000
--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.idx
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:16487669176cb9f4fea8940b9975db018056043b7b1b18787f785f4556b2f4da
-size 54179511
diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.pos b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.pos
deleted file mode 100644
index 8129de70..00000000
--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.pos
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f2244ce584650ca9fa2ec0e5b524e202eee54aef9ba542d5887b570661fdb5ee
-size 111341261
diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.store b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.store
deleted file mode 100644
index 1bb0fb7f..00000000
--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.store
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cf74f0db05175948a14bfa75151640a3eecb0f6b14d132fafd42b6ac76cde5fe
-size 108030942
diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.term b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.term
deleted file mode 100644
index d28c8a86..00000000
--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.term
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fdb1ab4a316df8f03330b45c860fb5885572924c6e67f77a05900180e29f3959
-size 85272549
diff --git a/data/index/inverted_index/meta.json b/data/index/inverted_index/meta.json
deleted file mode 100644
index af0b804b..00000000
--- a/data/index/inverted_index/meta.json
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f1e90f4610f7a4b67e128a704b3526865bb6e40647b18b2ec8b9a00701bdd74a
-size 4541
diff --git a/data/index/primary_image_store/000009.sst b/data/index/primary_image_store/000009.sst
deleted file mode 100644
index 40bd0ec5..00000000
--- a/data/index/primary_image_store/000009.sst
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:72de98db67475e1c827a002cd6a088b1384991f7d8251210d9d76d36ea7e8de2
-size 26418284
diff --git a/data/index/primary_image_store/CURRENT b/data/index/primary_image_store/CURRENT
deleted file mode 100644
index 0e9ee32d..00000000
--- a/data/index/primary_image_store/CURRENT
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9c20c7d5e29035f91bef5b253fedeae4b6e37dea976d050b34e1819eca981750
-size 16
diff --git a/data/index/primary_image_store/IDENTITY b/data/index/primary_image_store/IDENTITY
deleted file mode 100644
index b40de693..00000000
--- a/data/index/primary_image_store/IDENTITY
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:682cb9d331739607356ef4da3af8fce9f8322b8a3db036dd9b7d678bae8968ac
-size 36
diff --git a/data/index/primary_image_store/LOCK b/data/index/primary_image_store/LOCK
deleted file mode 100644
index e69de29b..00000000
diff --git a/data/index/primary_image_store/LOG b/data/index/primary_image_store/LOG
deleted file mode 100644
index 026ced18..00000000
--- a/data/index/primary_image_store/LOG
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ed44b93c59fd021438082141ae758cb4a767f0e1bf2ec426c87400970b1024fa
-size 19820
diff --git a/data/index/primary_image_store/MANIFEST-000058 b/data/index/primary_image_store/MANIFEST-000058
deleted file mode 100644
index 03fafbf4..00000000
--- a/data/index/primary_image_store/MANIFEST-000058
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e47f4d4510e56853a0ab3e3dbedbb88219ac20a5f6a13ee22ef8574d6151166b
-size 202
diff --git a/data/index/primary_image_store/MANIFEST-000158 b/data/index/primary_image_store/MANIFEST-000158
deleted file mode 100644
index 055ab21c..00000000
--- a/data/index/primary_image_store/MANIFEST-000158
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4bffb2c4239404184d24345fe9133939171338238c79e38b190d2e06244ebe66
-size 205
diff --git a/data/index/primary_image_store/OPTIONS-000056 b/data/index/primary_image_store/OPTIONS-000056
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/index/primary_image_store/OPTIONS-000056
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/primary_image_store/OPTIONS-000061 b/data/index/primary_image_store/OPTIONS-000061
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/index/primary_image_store/OPTIONS-000061
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/primary_image_store/OPTIONS-000156 b/data/index/primary_image_store/OPTIONS-000156
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/index/primary_image_store/OPTIONS-000156
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/primary_image_store/dictionary b/data/index/primary_image_store/dictionary
deleted file mode 100644
index 2452fa9d..00000000
--- a/data/index/primary_image_store/dictionary
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ffc57eb861fab46f5c16cffbe9a692a1beb072a1282bd325562a3a90928b9367
-size 5141310
diff --git a/justfile b/justfile
index b4f10bc0..18e4485c 100644
--- a/justfile
+++ b/justfile
@@ -1,3 +1,6 @@
+@unpack-data:
+ tar -zxvf data.tar.gz
+
@flamegraph:
sudo rm -rf data/index
cargo flamegraph --root -- indexer local configs/indexer/local.toml
@@ -21,3 +24,7 @@
@entity:
rm -rf data/entity
cargo run --release -- indexer entity data/enwiki-20220801-pages-articles-multistream.xml.bz2 data/entity
+
+@pack-data:
+ rm -f data.tar.gz
+ tar --exclude="data/enwiki*" --exclude="data/warc_files" --exclude="data/webgraph" -zcvf data.tar.gz data