web search done right
New strategy for git-lfs structure. RocksDb changes it's index each time it is opened which leads to many uploads to lfs. Also for some reason, the index in lfs seems to be broken.
Mikkel Denkere313b1c6cc95
diff --git a/.gitattributes b/.gitattributesindex 20739a9d..f6bd2897 100644--- a/.gitattributes+++ b/.gitattributes@@ -1 +1 @@-data/** filter=lfs diff=lfs merge=lfs -text+data.tar.gz filter=lfs diff=lfs merge=lfs -textdiff --git a/.gitignore b/.gitignoreindex ba941e4a..1ae26420 100644--- a/.gitignore+++ b/.gitignore@@ -1,15 +1,9 @@/target.DS_Storeflamegraph.svg-/data/enwiki-20220801-pages-articles-multistream.xml.bz2-/data/warc_files/CC-MAIN-20220116093137-20220116123137-00000.warc.gz-/data/warc_files/CC-MAIN-20220116093137-20220116123137-00001.warc.gz-/data/warc_files/CC-MAIN-20220116093137-20220116123137-00003.warc.gzstatic/**/*.cssnode_modulespackage-lock.jsonnotes*.log-/data/**/LOG.old*-/data/webgraph-/data/warc_files+/data/**/*diff --git a/data.tar.gz b/data.tar.gznew file mode 100644index 00000000..970d26d3--- /dev/null+++ b/data.tar.gz@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:2cabcd57754add6d968fb719d9c4e8a6d259a758995163027f7714ffec1e35d4+size 941862585diff --git a/data/.gitkeep b/data/.gitkeepdeleted file mode 100644index e69de29b..00000000diff --git a/data/centrality/conf b/data/centrality/confdeleted file mode 100755index 89002549..00000000--- a/data/centrality/conf+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:88a3aa53a63fd1728340d571e01a7983c9deb39be1b285b59ea6fad3adc052d8-size 61diff --git a/data/centrality/db b/data/centrality/dbdeleted file mode 100755index 6138769b..00000000--- a/data/centrality/db+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:f56e6255edea4fa061f88789c5b5b1b0a26b0d0b3ebb7f5955a5c10276ca329e-size 42467328diff --git a/data/centrality/snap.0000000009100BE4 b/data/centrality/snap.0000000009100BE4deleted file mode 100755index 36c3e9a8..00000000--- a/data/centrality/snap.0000000009100BE4+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:2063ab43eb74a396cbd2ad6b67c00c32230b580a29c37032e0de1c5fd6cf9d00-size 218377diff --git a/data/entity/attribute_occurrences/000009.sst b/data/entity/attribute_occurrences/000009.sstdeleted file mode 100644index 0267052c..00000000--- a/data/entity/attribute_occurrences/000009.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:4f56c95c5d31f8802c40f599ba4508fb6ef0ffffe6baf5b3e144cce938e4c3bb-size 174847diff --git a/data/entity/attribute_occurrences/000011.sst b/data/entity/attribute_occurrences/000011.sstdeleted file mode 100644index ba0a86f0..00000000--- a/data/entity/attribute_occurrences/000011.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:65270d2bfbe4aa90f7b5f583dd6755c9a47418a85de33c9eeda11b94e439b1b9-size 226925diff --git a/data/entity/attribute_occurrences/000013.sst b/data/entity/attribute_occurrences/000013.sstdeleted file mode 100644index 6edb961e..00000000--- a/data/entity/attribute_occurrences/000013.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:83717d6b717dbc2254b7c09130d888f898851e3eec62e1a541aed78da9d24b06-size 146557diff --git a/data/entity/attribute_occurrences/CURRENT b/data/entity/attribute_occurrences/CURRENTdeleted file mode 100644index 194192bf..00000000--- a/data/entity/attribute_occurrences/CURRENT+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:0c27ed6e0a0e47b52cf24da720967bfa648a86af64dbaed7f0248a9d0cc94b66-size 16diff --git a/data/entity/attribute_occurrences/IDENTITY b/data/entity/attribute_occurrences/IDENTITYdeleted file mode 100644index 72599f90..00000000--- a/data/entity/attribute_occurrences/IDENTITY+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:138fd3d1d88f180b677bbfc64d80d3c4d96de802d58da72f5f7ac0856a623d71-size 36diff --git a/data/entity/attribute_occurrences/LOCK b/data/entity/attribute_occurrences/LOCKdeleted file mode 100644index e69de29b..00000000diff --git a/data/entity/attribute_occurrences/LOG b/data/entity/attribute_occurrences/LOGdeleted file mode 100644index 846d9204..00000000--- a/data/entity/attribute_occurrences/LOG+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:fb02808941c8a51f3f76379e1db7e89dbcddd9266ffbcb2ce65f939269fc7f86-size 22983diff --git a/data/entity/attribute_occurrences/MANIFEST-000057 b/data/entity/attribute_occurrences/MANIFEST-000057deleted file mode 100644index 18d28e27..00000000--- a/data/entity/attribute_occurrences/MANIFEST-000057+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:fb818c54efcaf96a16becbe4eed72e4bda32e4f81fb57cb1b5426afe6374ff8b-size 428diff --git a/data/entity/attribute_occurrences/MANIFEST-000132 b/data/entity/attribute_occurrences/MANIFEST-000132deleted file mode 100644index edc6424e..00000000--- a/data/entity/attribute_occurrences/MANIFEST-000132+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:0eb8f9fb6aa535b0d3e68cc1e8648334cc04f65ddacab57371416c87ef64d270-size 430diff --git a/data/entity/attribute_occurrences/OPTIONS-000055 b/data/entity/attribute_occurrences/OPTIONS-000055deleted file mode 100644index 9f62adb9..00000000--- a/data/entity/attribute_occurrences/OPTIONS-000055+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/entity/attribute_occurrences/OPTIONS-000060 b/data/entity/attribute_occurrences/OPTIONS-000060deleted file mode 100644index 9f62adb9..00000000--- a/data/entity/attribute_occurrences/OPTIONS-000060+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/entity/attribute_occurrences/OPTIONS-000130 b/data/entity/attribute_occurrences/OPTIONS-000130deleted file mode 100644index 9f62adb9..00000000--- a/data/entity/attribute_occurrences/OPTIONS-000130+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/entity/images/000016.sst b/data/entity/images/000016.sstdeleted file mode 100644index f98d586a..00000000--- a/data/entity/images/000016.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:eff6efc0f5d63961e6c67b364bcc00cb9124cf51a889a68560bed3f7d6b0d0d8-size 67218210diff --git a/data/entity/images/000017.sst b/data/entity/images/000017.sstdeleted file mode 100644index 5b6e39ec..00000000--- a/data/entity/images/000017.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:1f3d794beb951cf3ef2bf560c723e9d354dfffc36cb5499db9e95bdb905d2b73-size 67310472diff --git a/data/entity/images/000018.sst b/data/entity/images/000018.sstdeleted file mode 100644index 0814d4b4..00000000--- a/data/entity/images/000018.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:a74f57be3610ce8b4c223daf1063614d4e576b7001e0c6dad2bf8d598e36e622-size 67213493diff --git a/data/entity/images/000019.sst b/data/entity/images/000019.sstdeleted file mode 100644index a3bf0d68..00000000--- a/data/entity/images/000019.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:ab1d15dd82ac67c1c8ab3adf097bd06f81ea5207342029ebcc73d6245c379abc-size 60042860diff --git a/data/entity/images/000021.sst b/data/entity/images/000021.sstdeleted file mode 100644index 9cbe93b0..00000000--- a/data/entity/images/000021.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9a151c8f0d006c1223b76aebc14120212c309daae346b6f0a08d1547ffec7077-size 65661368diff --git a/data/entity/images/000023.sst b/data/entity/images/000023.sstdeleted file mode 100644index 7fb7f067..00000000--- a/data/entity/images/000023.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:d480a9623fcb9a4e9115a94bc8643548bd6ec659b395d93dc23edba8d57572f6-size 61004478diff --git a/data/entity/images/CURRENT b/data/entity/images/CURRENTdeleted file mode 100644index 4c549b93..00000000--- a/data/entity/images/CURRENT+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:eae9fea6aa6c724238caf1cd3b0fbd967fa9f57afeccc437e3c0e5d563891ee7-size 16diff --git a/data/entity/images/IDENTITY b/data/entity/images/IDENTITYdeleted file mode 100644index 1e467612..00000000--- a/data/entity/images/IDENTITY+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:1689b459ee8a2aea971605f2f99f4742cda84f08e248afe84161e6e6c9500b12-size 36diff --git a/data/entity/images/LOCK b/data/entity/images/LOCKdeleted file mode 100644index e69de29b..00000000diff --git a/data/entity/images/LOG b/data/entity/images/LOGdeleted file mode 100644index da9cecd0..00000000--- a/data/entity/images/LOG+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:8cddc9638f2d59767edba2e54372b73581dc06690f9f4e7fd9e5d4e86a23bcdf-size 23176diff --git a/data/entity/images/MANIFEST-000067 b/data/entity/images/MANIFEST-000067deleted file mode 100644index d99b90ee..00000000--- a/data/entity/images/MANIFEST-000067+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:e0429f925c46995a24c552aa3695b00a41634a38fc7e5d4b7ec0f779b7bd5062-size 775diff --git a/data/entity/images/MANIFEST-000144 b/data/entity/images/MANIFEST-000144deleted file mode 100644index 43a370c6..00000000--- a/data/entity/images/MANIFEST-000144+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:012a5cc96e37a41a51166cbf5bdc2c0e0995637094efef61df6a2d27ac7ca2ac-size 910diff --git a/data/entity/images/OPTIONS-000065 b/data/entity/images/OPTIONS-000065deleted file mode 100644index 9f62adb9..00000000--- a/data/entity/images/OPTIONS-000065+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/entity/images/OPTIONS-000070 b/data/entity/images/OPTIONS-000070deleted file mode 100644index 9f62adb9..00000000--- a/data/entity/images/OPTIONS-000070+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/entity/images/OPTIONS-000142 b/data/entity/images/OPTIONS-000142deleted file mode 100644index 9f62adb9..00000000--- a/data/entity/images/OPTIONS-000142+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/entity/inverted_index/.managed.json b/data/entity/inverted_index/.managed.jsondeleted file mode 100644index 9b1db3ae..00000000--- a/data/entity/inverted_index/.managed.json+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:3601da667cad6bb25d5ce6fe19c37d510e815b8c18ffe795e86b94f377035930-size 258diff --git a/data/entity/inverted_index/.tantivy-meta.lock b/data/entity/inverted_index/.tantivy-meta.lockdeleted file mode 100644index e69de29b..00000000diff --git a/data/entity/inverted_index/.tantivy-writer.lock b/data/entity/inverted_index/.tantivy-writer.lockdeleted file mode 100644index e69de29b..00000000diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fast b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fastdeleted file mode 100644index 7f089e2b..00000000--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fast+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:b253ab4389b9114e0d15974570866464bccebfcc29ba6a1f79f3808dbef25443-size 99diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fieldnorm b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fieldnormdeleted file mode 100644index d6a5a497..00000000--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.fieldnorm+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:8b75ed3d24a4a59938e97561c1c2f5613ebeb4fa806022ae1e85708a56640be2-size 600121diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.idx b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.idxdeleted file mode 100644index d2a593f8..00000000--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.idx+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:c27203416473e02b21a4b1e38135108d2c8b82a1a83dd3dfd4484afb0a50d388-size 26057756diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.pos b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.posdeleted file mode 100644index cd63e56a..00000000--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.pos+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:7d5c550f6cfeb8508ddbde9aa741a1ab225ba1595c36a364b24291dbaa665ad2-size 20506426diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.store b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.storedeleted file mode 100644index e5c2c1db..00000000--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.store+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:f45131394b38d9c917f8b725c98ec80ae1967c60dd1f027b06bc0fda647476c1-size 237946448diff --git a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.term b/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.termdeleted file mode 100644index ca3ea7bd..00000000--- a/data/entity/inverted_index/cb022cd175e04430a93d823af22ffd48.term+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:c85d71c7961a3ae4c559661e4920c2bfd0cc97845f4b416f76765034bbeafc0b-size 5387812diff --git a/data/entity/inverted_index/meta.json b/data/entity/inverted_index/meta.jsondeleted file mode 100644index 62a38409..00000000--- a/data/entity/inverted_index/meta.json+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:7ae6733e1cb2efe06d6aeb3aea06d1face0ffa32cf835efaf770d91765009763-size 1437diff --git a/data/index/favicon_store/000009.sst b/data/index/favicon_store/000009.sstdeleted file mode 100644index ffdacd79..00000000--- a/data/index/favicon_store/000009.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:bff70c998a5f6398310c53d1427156c19a0536c99f29696087171182ef991d5d-size 130482diff --git a/data/index/favicon_store/CURRENT b/data/index/favicon_store/CURRENTdeleted file mode 100644index 0e9ee32d..00000000--- a/data/index/favicon_store/CURRENT+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9c20c7d5e29035f91bef5b253fedeae4b6e37dea976d050b34e1819eca981750-size 16diff --git a/data/index/favicon_store/IDENTITY b/data/index/favicon_store/IDENTITYdeleted file mode 100644index d9615944..00000000--- a/data/index/favicon_store/IDENTITY+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:e62e2a4e1c3c3514f88e78a47b831d082897ea7bf3dd1d7cd64b292431e3eb3a-size 36diff --git a/data/index/favicon_store/LOCK b/data/index/favicon_store/LOCKdeleted file mode 100644index e69de29b..00000000diff --git a/data/index/favicon_store/LOG b/data/index/favicon_store/LOGdeleted file mode 100644index cc01fb57..00000000--- a/data/index/favicon_store/LOG+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:e003c3f153939ed9ed29025b30af57059f20629d3c5f937397eaae3bb759263a-size 19788diff --git a/data/index/favicon_store/MANIFEST-000058 b/data/index/favicon_store/MANIFEST-000058deleted file mode 100644index f1783b02..00000000--- a/data/index/favicon_store/MANIFEST-000058+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:409accc944bd2c8a32b7796a43192dc10e1b7005580497a65c1315a218a6a3e1-size 173diff --git a/data/index/favicon_store/MANIFEST-000158 b/data/index/favicon_store/MANIFEST-000158deleted file mode 100644index 1f7a0b8a..00000000--- a/data/index/favicon_store/MANIFEST-000158+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:28ab7a432c135fabb46de3ea420fb8d21d5abb9a47e5879748bfffd9f9f115f0-size 176diff --git a/data/index/favicon_store/OPTIONS-000056 b/data/index/favicon_store/OPTIONS-000056deleted file mode 100644index 9f62adb9..00000000--- a/data/index/favicon_store/OPTIONS-000056+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/favicon_store/OPTIONS-000061 b/data/index/favicon_store/OPTIONS-000061deleted file mode 100644index 9f62adb9..00000000--- a/data/index/favicon_store/OPTIONS-000061+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/favicon_store/OPTIONS-000156 b/data/index/favicon_store/OPTIONS-000156deleted file mode 100644index 9f62adb9..00000000--- a/data/index/favicon_store/OPTIONS-000156+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/inverted_index/.managed.json b/data/index/inverted_index/.managed.jsondeleted file mode 100644index 6cc3f9cf..00000000--- a/data/index/inverted_index/.managed.json+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:8255799db4e3bd16e19fd14bdf9c6612b795f0853aa4ca06634088e2cf8927d6-size 258diff --git a/data/index/inverted_index/.tantivy-meta.lock b/data/index/inverted_index/.tantivy-meta.lockdeleted file mode 100644index e69de29b..00000000diff --git a/data/index/inverted_index/.tantivy-writer.lock b/data/index/inverted_index/.tantivy-writer.lockdeleted file mode 100644index e69de29b..00000000diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fast b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fastdeleted file mode 100644index 80cbe8b7..00000000--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fast+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:a15c7a425ed19862955c091f08ca6f2f72c06d52d9c4c6e659b807ee12557159-size 229028diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fieldnorm b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fieldnormdeleted file mode 100644index c875d09c..00000000--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.fieldnorm+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:7c084af14222dba37866f2a51700306bb324efc3243acfa43112544155a506a1-size 318675diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.idx b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.idxdeleted file mode 100644index bba4358a..00000000--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.idx+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:16487669176cb9f4fea8940b9975db018056043b7b1b18787f785f4556b2f4da-size 54179511diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.pos b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.posdeleted file mode 100644index 8129de70..00000000--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.pos+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:f2244ce584650ca9fa2ec0e5b524e202eee54aef9ba542d5887b570661fdb5ee-size 111341261diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.store b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.storedeleted file mode 100644index 1bb0fb7f..00000000--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.store+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:cf74f0db05175948a14bfa75151640a3eecb0f6b14d132fafd42b6ac76cde5fe-size 108030942diff --git a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.term b/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.termdeleted file mode 100644index d28c8a86..00000000--- a/data/index/inverted_index/fdf92d284e0f45efbc14a89da3693e6f.term+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:fdb1ab4a316df8f03330b45c860fb5885572924c6e67f77a05900180e29f3959-size 85272549diff --git a/data/index/inverted_index/meta.json b/data/index/inverted_index/meta.jsondeleted file mode 100644index af0b804b..00000000--- a/data/index/inverted_index/meta.json+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:f1e90f4610f7a4b67e128a704b3526865bb6e40647b18b2ec8b9a00701bdd74a-size 4541diff --git a/data/index/primary_image_store/000009.sst b/data/index/primary_image_store/000009.sstdeleted file mode 100644index 40bd0ec5..00000000--- a/data/index/primary_image_store/000009.sst+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:72de98db67475e1c827a002cd6a088b1384991f7d8251210d9d76d36ea7e8de2-size 26418284diff --git a/data/index/primary_image_store/CURRENT b/data/index/primary_image_store/CURRENTdeleted file mode 100644index 0e9ee32d..00000000--- a/data/index/primary_image_store/CURRENT+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9c20c7d5e29035f91bef5b253fedeae4b6e37dea976d050b34e1819eca981750-size 16diff --git a/data/index/primary_image_store/IDENTITY b/data/index/primary_image_store/IDENTITYdeleted file mode 100644index b40de693..00000000--- a/data/index/primary_image_store/IDENTITY+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:682cb9d331739607356ef4da3af8fce9f8322b8a3db036dd9b7d678bae8968ac-size 36diff --git a/data/index/primary_image_store/LOCK b/data/index/primary_image_store/LOCKdeleted file mode 100644index e69de29b..00000000diff --git a/data/index/primary_image_store/LOG b/data/index/primary_image_store/LOGdeleted file mode 100644index 026ced18..00000000--- a/data/index/primary_image_store/LOG+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:ed44b93c59fd021438082141ae758cb4a767f0e1bf2ec426c87400970b1024fa-size 19820diff --git a/data/index/primary_image_store/MANIFEST-000058 b/data/index/primary_image_store/MANIFEST-000058deleted file mode 100644index 03fafbf4..00000000--- a/data/index/primary_image_store/MANIFEST-000058+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:e47f4d4510e56853a0ab3e3dbedbb88219ac20a5f6a13ee22ef8574d6151166b-size 202diff --git a/data/index/primary_image_store/MANIFEST-000158 b/data/index/primary_image_store/MANIFEST-000158deleted file mode 100644index 055ab21c..00000000--- a/data/index/primary_image_store/MANIFEST-000158+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:4bffb2c4239404184d24345fe9133939171338238c79e38b190d2e06244ebe66-size 205diff --git a/data/index/primary_image_store/OPTIONS-000056 b/data/index/primary_image_store/OPTIONS-000056deleted file mode 100644index 9f62adb9..00000000--- a/data/index/primary_image_store/OPTIONS-000056+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/primary_image_store/OPTIONS-000061 b/data/index/primary_image_store/OPTIONS-000061deleted file mode 100644index 9f62adb9..00000000--- a/data/index/primary_image_store/OPTIONS-000061+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/primary_image_store/OPTIONS-000156 b/data/index/primary_image_store/OPTIONS-000156deleted file mode 100644index 9f62adb9..00000000--- a/data/index/primary_image_store/OPTIONS-000156+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/primary_image_store/dictionary b/data/index/primary_image_store/dictionarydeleted file mode 100644index 2452fa9d..00000000--- a/data/index/primary_image_store/dictionary+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:ffc57eb861fab46f5c16cffbe9a692a1beb072a1282bd325562a3a90928b9367-size 5141310diff --git a/justfile b/justfileindex b4f10bc0..18e4485c 100644--- a/justfile+++ b/justfile@@ -1,3 +1,6 @@+@unpack-data:+ tar -zxvf data.tar.gz+@flamegraph:sudo rm -rf data/indexcargo flamegraph --root -- indexer local configs/indexer/local.toml@@ -21,3 +24,7 @@@entity:rm -rf data/entitycargo run --release -- indexer entity data/enwiki-20220801-pages-articles-multistream.xml.bz2 data/entity++@pack-data:+ rm -f data.tar.gz+ tar --exclude="data/enwiki*" --exclude="data/warc_files" --exclude="data/webgraph" -zcvf data.tar.gz data