Skip to content

World's first open-source enterprise world model.

Text read from a PDF uses the characters it shows, not the radicals and ligatures that look like them (#1013)

Maya Kiddef573c99f45
diff --git a/crates/utopia-ingest/src/EquivalentUnifiedIdeograph.txt b/crates/utopia-ingest/src/EquivalentUnifiedIdeograph.txt
new file mode 100644
index 00000000..1a70d6a6
--- /dev/null
+++ b/crates/utopia-ingest/src/EquivalentUnifiedIdeograph.txt
@@ -0,0 +1,410 @@
+# EquivalentUnifiedIdeograph-18.0.0.txt
+# Date: 2026-02-03
+# © 2026 Unicode®, Inc.
+# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
+#
+# For documentation, see UAX #44: Unicode Character Database,
+# at https://www.unicode.org/reports/tr44/
+#
+# This file defines the following property:
+#
+# Equivalent_Unified_Ideograph
+#
+# This data file lists CJK radicals and CJK strokes (aka characters
+# that have the Radical=Yes property or are in the CJK Strokes block)
+# for which there are reasonably equivalent CJK unified ideographs in
+# that they are visually identical or near-identical.
+#
+# The file contains a list of lines with mappings from one code point
+# or code point range to an equivalent CJK unified ideograph.
+#
+# Each mapping line contains two fields, separated by a semicolon (';').
+# The first field contains a code point or code point range represented
+# as a variable-length hexadecimal value with 4 to 6 digits. The second
+# field contains a code point represented as a variable-length
+# hexadecimal value with 4 to 6 digits.
+#
+# CJK radicals and CJK strokes for which no reasonably equivalent CJK
+# unified ideographs exist are listed as comments at the end of the file.
+#
+# Formally, the default value of the Equivalent_Unified_Ideograph
+# property for each code point is <none>, unless a mapping to some
+# other character is specified in this data file. When a code point
+# has the default value for the Equivalent_Unified_Ideograph property,
+# that means that no other character exists whose representative glyph
+# is a reasonably equivalent CJK unified ideograph.
+#
+# This file was originally created by Ken Lunde.
+#
+# ############################################################
+#
+# Property: Equivalent_Unified_Ideograph
+#
+# @missing: 0000..10FFFF; <none>
+
+2E81 ; 5382 # CJK RADICAL CLIFF
+2E82 ; 4E5B # CJK RADICAL SECOND ONE
+2E83 ; 4E5A # CJK RADICAL SECOND TWO
+2E84 ; 4E59 # CJK RADICAL SECOND THREE
+2E85 ; 4EBB # CJK RADICAL PERSON
+2E86 ; 5182 # CJK RADICAL BOX
+2E87 ; 20628 # CJK RADICAL TABLE
+2E88 ; 5200 # CJK RADICAL KNIFE ONE
+2E89 ; 5202 # CJK RADICAL KNIFE TWO
+2E8A ; 535C # CJK RADICAL DIVINATION
+2E8B ; 353E # CJK RADICAL SEAL
+2E8C..2E8D ; 5C0F # [2] CJK RADICAL SMALL ONE..CJK RADICAL SMALL TWO
+2E8E ; 5140 # CJK RADICAL LAME ONE
+2E8F ; 5C23 # CJK RADICAL LAME TWO
+2E90 ; 5C22 # CJK RADICAL LAME THREE
+2E91 ; 21BC2 # CJK RADICAL LAME FOUR
+2E92 ; 5DF3 # CJK RADICAL SNAKE
+2E93 ; 5E7A # CJK RADICAL THREAD
+2E94 ; 5F51 # CJK RADICAL SNOUT ONE
+2E95 ; 2B739 # CJK RADICAL SNOUT TWO
+2E96 ; 5FC4 # CJK RADICAL HEART ONE
+2E97 ; 5FC3 # CJK RADICAL HEART TWO
+2E98 ; 624C # CJK RADICAL HAND
+2E99 ; 6535 # CJK RADICAL RAP
+2E9B ; 65E1 # CJK RADICAL CHOKE
+2E9C ; 65E5 # CJK RADICAL SUN
+2E9D ; 6708 # CJK RADICAL MOON
+2E9E ; 6B7A # CJK RADICAL DEATH
+2E9F ; 6BCD # CJK RADICAL MOTHER
+2EA0 ; 6C11 # CJK RADICAL CIVILIAN
+2EA1 ; 6C35 # CJK RADICAL WATER ONE
+2EA2 ; 6C3A # CJK RADICAL WATER TWO
+2EA3 ; 706C # CJK RADICAL FIRE
+2EA4..2EA5 ; 722B # [2] CJK RADICAL PAW ONE..CJK RADICAL PAW TWO
+2EA6 ; 4E2C # CJK RADICAL SIMPLIFIED HALF TREE TRUNK
+2EA7 ; 725B # CJK RADICAL COW
+2EA8 ; 72AD # CJK RADICAL DOG
+2EA9 ; 738B # CJK RADICAL JADE
+2EAA ; 24D14 # CJK RADICAL BOLT OF CLOTH
+2EAB ; 76EE # CJK RADICAL EYE
+2EAC ; 793A # CJK RADICAL SPIRIT ONE
+2EAD ; 793B # CJK RADICAL SPIRIT TWO
+2EAE ; 25AD7 # CJK RADICAL BAMBOO
+2EAF ; 7CF9 # CJK RADICAL SILK
+2EB0 ; 7E9F # CJK RADICAL C-SIMPLIFIED SILK
+2EB1 ; 7F53 # CJK RADICAL NET ONE
+2EB2 ; 7F52 # CJK RADICAL NET TWO
+2EB3 ; 34C1 # CJK RADICAL NET THREE
+2EB4 ; 5197 # CJK RADICAL NET FOUR
+2EB5 ; 2626B # CJK RADICAL MESH
+2EB6 ; 7F8A # CJK RADICAL SHEEP
+2EB7 ; 2634C # CJK RADICAL RAM
+2EB8 ; 2634B # CJK RADICAL EWE
+2EB9 ; 8002 # CJK RADICAL OLD
+2EBA ; 8080 # CJK RADICAL BRUSH ONE
+2EBB ; 807F # CJK RADICAL BRUSH TWO
+2EBC ; 8089 # CJK RADICAL MEAT
+2EBD ; 26951 # CJK RADICAL MORTAR
+2EBE..2EC0 ; 8279 # [3] CJK RADICAL GRASS ONE..CJK RADICAL GRASS THREE
+2EC1 ; 864E # CJK RADICAL TIGER
+2EC2 ; 8864 # CJK RADICAL CLOTHES
+2EC3 ; 8980 # CJK RADICAL WEST ONE
+2EC4 ; 897F # CJK RADICAL WEST TWO
+2EC5 ; 89C1 # CJK RADICAL C-SIMPLIFIED SEE
+2EC6 ; 89D2 # CJK RADICAL SIMPLIFIED HORN
+2EC7 ; 278B2 # CJK RADICAL HORN
+2EC8 ; 8BA0 # CJK RADICAL C-SIMPLIFIED SPEECH
+2EC9 ; 8D1D # CJK RADICAL C-SIMPLIFIED SHELL
+2ECA ; 27FB7 # CJK RADICAL FOOT
+2ECB ; 8F66 # CJK RADICAL C-SIMPLIFIED CART
+2ECC..2ECE ; 8FB6 # [3] CJK RADICAL SIMPLIFIED WALK..CJK RADICAL WALK TWO
+2ECF ; 9091 # CJK RADICAL CITY
+2ED0 ; 9485 # CJK RADICAL C-SIMPLIFIED GOLD
+2ED1 ; 9577 # CJK RADICAL LONG ONE
+2ED2 ; 9578 # CJK RADICAL LONG TWO
+2ED3 ; 957F # CJK RADICAL C-SIMPLIFIED LONG
+2ED4 ; 95E8 # CJK RADICAL C-SIMPLIFIED GATE
+2ED5 ; 28E0F # CJK RADICAL MOUND ONE
+2ED6 ; 961D # CJK RADICAL MOUND TWO
+2ED7 ; 96E8 # CJK RADICAL RAIN
+2ED8 ; 9752 # CJK RADICAL BLUE
+2ED9 ; 97E6 # CJK RADICAL C-SIMPLIFIED TANNED LEATHER
+2EDA ; 9875 # CJK RADICAL C-SIMPLIFIED LEAF
+2EDB ; 98CE # CJK RADICAL C-SIMPLIFIED WIND
+2EDC ; 98DE # CJK RADICAL C-SIMPLIFIED FLY
+2EDD ; 98DF # CJK RADICAL EAT ONE
+2EDE ; 2967F # CJK RADICAL EAT TWO
+2EDF ; 98E0 # CJK RADICAL EAT THREE
+2EE0 ; 9963 # CJK RADICAL C-SIMPLIFIED EAT
+2EE1 ; 29810 # CJK RADICAL HEAD
+2EE2 ; 9A6C # CJK RADICAL C-SIMPLIFIED HORSE
+2EE3 ; 9AA8 # CJK RADICAL BONE
+2EE4 ; 9B3C # CJK RADICAL GHOST
+2EE5 ; 9C7C # CJK RADICAL C-SIMPLIFIED FISH
+2EE6 ; 9E1F # CJK RADICAL C-SIMPLIFIED BIRD
+2EE7 ; 5364 # CJK RADICAL C-SIMPLIFIED SALT
+2EE8 ; 9EA6 # CJK RADICAL SIMPLIFIED WHEAT
+2EE9 ; 9EC4 # CJK RADICAL SIMPLIFIED YELLOW
+2EEA ; 9EFE # CJK RADICAL C-SIMPLIFIED FROG
+2EEB ; 6589 # CJK RADICAL J-SIMPLIFIED EVEN
+2EEC ; 9F50 # CJK RADICAL C-SIMPLIFIED EVEN
+2EED ; 6B6F # CJK RADICAL J-SIMPLIFIED TOOTH
+2EEE ; 9F7F # CJK RADICAL C-SIMPLIFIED TOOTH
+2EEF ; 7ADC # CJK RADICAL J-SIMPLIFIED DRAGON
+2EF0 ; 9F99 # CJK RADICAL C-SIMPLIFIED DRAGON
+2EF1 ; 9F9C # CJK RADICAL TURTLE
+2EF2 ; 4E80 # CJK RADICAL J-SIMPLIFIED TURTLE
+2EF3 ; 9F9F # CJK RADICAL C-SIMPLIFIED TURTLE
+2F00 ; 4E00 # KANGXI RADICAL ONE
+2F01 ; 4E28 # KANGXI RADICAL LINE
+2F02 ; 4E36 # KANGXI RADICAL DOT
+2F03 ; 4E3F # KANGXI RADICAL SLASH
+2F04 ; 4E59 # KANGXI RADICAL SECOND
+2F05 ; 4E85 # KANGXI RADICAL HOOK
+2F06 ; 4E8C # KANGXI RADICAL TWO
+2F07 ; 4EA0 # KANGXI RADICAL LID
+2F08 ; 4EBA # KANGXI RADICAL MAN
+2F09 ; 513F # KANGXI RADICAL LEGS
+2F0A ; 5165 # KANGXI RADICAL ENTER
+2F0B ; 516B # KANGXI RADICAL EIGHT
+2F0C ; 5182 # KANGXI RADICAL DOWN BOX
+2F0D ; 5196 # KANGXI RADICAL COVER
+2F0E ; 51AB # KANGXI RADICAL ICE
+2F0F ; 51E0 # KANGXI RADICAL TABLE
+2F10 ; 51F5 # KANGXI RADICAL OPEN BOX
+2F11 ; 5200 # KANGXI RADICAL KNIFE
+2F12 ; 529B # KANGXI RADICAL POWER
+2F13 ; 52F9 # KANGXI RADICAL WRAP
+2F14 ; 5315 # KANGXI RADICAL SPOON
+2F15 ; 531A # KANGXI RADICAL RIGHT OPEN BOX
+2F16 ; 5338 # KANGXI RADICAL HIDING ENCLOSURE
+2F17 ; 5341 # KANGXI RADICAL TEN
+2F18 ; 535C # KANGXI RADICAL DIVINATION
+2F19 ; 5369 # KANGXI RADICAL SEAL
+2F1A ; 5382 # KANGXI RADICAL CLIFF
+2F1B ; 53B6 # KANGXI RADICAL PRIVATE
+2F1C ; 53C8 # KANGXI RADICAL AGAIN
+2F1D ; 53E3 # KANGXI RADICAL MOUTH
+2F1E ; 56D7 # KANGXI RADICAL ENCLOSURE
+2F1F ; 571F # KANGXI RADICAL EARTH
+2F20 ; 58EB # KANGXI RADICAL SCHOLAR
+2F21 ; 5902 # KANGXI RADICAL GO
+2F22 ; 590A # KANGXI RADICAL GO SLOWLY
+2F23 ; 5915 # KANGXI RADICAL EVENING
+2F24 ; 5927 # KANGXI RADICAL BIG
+2F25 ; 5973 # KANGXI RADICAL WOMAN
+2F26 ; 5B50 # KANGXI RADICAL CHILD
+2F27 ; 5B80 # KANGXI RADICAL ROOF
+2F28 ; 5BF8 # KANGXI RADICAL INCH
+2F29 ; 5C0F # KANGXI RADICAL SMALL
+2F2A ; 5C22 # KANGXI RADICAL LAME
+2F2B ; 5C38 # KANGXI RADICAL CORPSE
+2F2C ; 5C6E # KANGXI RADICAL SPROUT
+2F2D ; 5C71 # KANGXI RADICAL MOUNTAIN
+2F2E ; 5DDB # KANGXI RADICAL RIVER
+2F2F ; 5DE5 # KANGXI RADICAL WORK
+2F30 ; 5DF1 # KANGXI RADICAL ONESELF
+2F31 ; 5DFE # KANGXI RADICAL TURBAN
+2F32 ; 5E72 # KANGXI RADICAL DRY
+2F33 ; 5E7A # KANGXI RADICAL SHORT THREAD
+2F34 ; 5E7F # KANGXI RADICAL DOTTED CLIFF
+2F35 ; 5EF4 # KANGXI RADICAL LONG STRIDE
+2F36 ; 5EFE # KANGXI RADICAL TWO HANDS
+2F37 ; 5F0B # KANGXI RADICAL SHOOT
+2F38 ; 5F13 # KANGXI RADICAL BOW
+2F39 ; 5F50 # KANGXI RADICAL SNOUT
+2F3A ; 5F61 # KANGXI RADICAL BRISTLE
+2F3B ; 5F73 # KANGXI RADICAL STEP
+2F3C ; 5FC3 # KANGXI RADICAL HEART
+2F3D ; 6208 # KANGXI RADICAL HALBERD
+2F3E ; 6236 # KANGXI RADICAL DOOR
+2F3F ; 624B # KANGXI RADICAL HAND
+2F40 ; 652F # KANGXI RADICAL BRANCH
+2F41 ; 6534 # KANGXI RADICAL RAP
+2F42 ; 6587 # KANGXI RADICAL SCRIPT
+2F43 ; 6597 # KANGXI RADICAL DIPPER
+2F44 ; 65A4 # KANGXI RADICAL AXE
+2F45 ; 65B9 # KANGXI RADICAL SQUARE
+2F46 ; 65E0 # KANGXI RADICAL NOT
+2F47 ; 65E5 # KANGXI RADICAL SUN
+2F48 ; 66F0 # KANGXI RADICAL SAY
+2F49 ; 6708 # KANGXI RADICAL MOON
+2F4A ; 6728 # KANGXI RADICAL TREE
+2F4B ; 6B20 # KANGXI RADICAL LACK
+2F4C ; 6B62 # KANGXI RADICAL STOP
+2F4D ; 6B79 # KANGXI RADICAL DEATH
+2F4E ; 6BB3 # KANGXI RADICAL WEAPON
+2F4F ; 6BCB # KANGXI RADICAL DO NOT
+2F50 ; 6BD4 # KANGXI RADICAL COMPARE
+2F51 ; 6BDB # KANGXI RADICAL FUR
+2F52 ; 6C0F # KANGXI RADICAL CLAN
+2F53 ; 6C14 # KANGXI RADICAL STEAM
+2F54 ; 6C34 # KANGXI RADICAL WATER
+2F55 ; 706B # KANGXI RADICAL FIRE
+2F56 ; 722A # KANGXI RADICAL CLAW
+2F57 ; 7236 # KANGXI RADICAL FATHER
+2F58 ; 723B # KANGXI RADICAL DOUBLE X
+2F59 ; 723F # KANGXI RADICAL HALF TREE TRUNK
+2F5A ; 7247 # KANGXI RADICAL SLICE
+2F5B ; 7259 # KANGXI RADICAL FANG
+2F5C ; 725B # KANGXI RADICAL COW
+2F5D ; 72AC # KANGXI RADICAL DOG
+2F5E ; 7384 # KANGXI RADICAL PROFOUND
+2F5F ; 7389 # KANGXI RADICAL JADE
+2F60 ; 74DC # KANGXI RADICAL MELON
+2F61 ; 74E6 # KANGXI RADICAL TILE
+2F62 ; 7518 # KANGXI RADICAL SWEET
+2F63 ; 751F # KANGXI RADICAL LIFE
+2F64 ; 7528 # KANGXI RADICAL USE
+2F65 ; 7530 # KANGXI RADICAL FIELD
+2F66 ; 758B # KANGXI RADICAL BOLT OF CLOTH
+2F67 ; 7592 # KANGXI RADICAL SICKNESS
+2F68 ; 7676 # KANGXI RADICAL DOTTED TENT
+2F69 ; 767D # KANGXI RADICAL WHITE
+2F6A ; 76AE # KANGXI RADICAL SKIN
+2F6B ; 76BF # KANGXI RADICAL DISH
+2F6C ; 76EE # KANGXI RADICAL EYE
+2F6D ; 77DB # KANGXI RADICAL SPEAR
+2F6E ; 77E2 # KANGXI RADICAL ARROW
+2F6F ; 77F3 # KANGXI RADICAL STONE
+2F70 ; 793A # KANGXI RADICAL SPIRIT
+2F71 ; 79B8 # KANGXI RADICAL TRACK
+2F72 ; 79BE # KANGXI RADICAL GRAIN
+2F73 ; 7A74 # KANGXI RADICAL CAVE
+2F74 ; 7ACB # KANGXI RADICAL STAND
+2F75 ; 7AF9 # KANGXI RADICAL BAMBOO
+2F76 ; 7C73 # KANGXI RADICAL RICE
+2F77 ; 7CF8 # KANGXI RADICAL SILK
+2F78 ; 7F36 # KANGXI RADICAL JAR
+2F79 ; 7F51 # KANGXI RADICAL NET
+2F7A ; 7F8A # KANGXI RADICAL SHEEP
+2F7B ; 7FBD # KANGXI RADICAL FEATHER
+2F7C ; 8001 # KANGXI RADICAL OLD
+2F7D ; 800C # KANGXI RADICAL AND
+2F7E ; 8012 # KANGXI RADICAL PLOW
+2F7F ; 8033 # KANGXI RADICAL EAR
+2F80 ; 807F # KANGXI RADICAL BRUSH
+2F81 ; 8089 # KANGXI RADICAL MEAT
+2F82 ; 81E3 # KANGXI RADICAL MINISTER
+2F83 ; 81EA # KANGXI RADICAL SELF
+2F84 ; 81F3 # KANGXI RADICAL ARRIVE
+2F85 ; 81FC # KANGXI RADICAL MORTAR
+2F86 ; 820C # KANGXI RADICAL TONGUE
+2F87 ; 821B # KANGXI RADICAL OPPOSE
+2F88 ; 821F # KANGXI RADICAL BOAT
+2F89 ; 826E # KANGXI RADICAL STOPPING
+2F8A ; 8272 # KANGXI RADICAL COLOR
+2F8B ; 8278 # KANGXI RADICAL GRASS
+2F8C ; 864D # KANGXI RADICAL TIGER
+2F8D ; 866B # KANGXI RADICAL INSECT
+2F8E ; 8840 # KANGXI RADICAL BLOOD
+2F8F ; 884C # KANGXI RADICAL WALK ENCLOSURE
+2F90 ; 8863 # KANGXI RADICAL CLOTHES
+2F91 ; 897E # KANGXI RADICAL WEST
+2F92 ; 898B # KANGXI RADICAL SEE
+2F93 ; 89D2 # KANGXI RADICAL HORN
+2F94 ; 8A00 # KANGXI RADICAL SPEECH
+2F95 ; 8C37 # KANGXI RADICAL VALLEY
+2F96 ; 8C46 # KANGXI RADICAL BEAN
+2F97 ; 8C55 # KANGXI RADICAL PIG
+2F98 ; 8C78 # KANGXI RADICAL BADGER
+2F99 ; 8C9D # KANGXI RADICAL SHELL
+2F9A ; 8D64 # KANGXI RADICAL RED
+2F9B ; 8D70 # KANGXI RADICAL RUN
+2F9C ; 8DB3 # KANGXI RADICAL FOOT
+2F9D ; 8EAB # KANGXI RADICAL BODY
+2F9E ; 8ECA # KANGXI RADICAL CART
+2F9F ; 8F9B # KANGXI RADICAL BITTER
+2FA0 ; 8FB0 # KANGXI RADICAL MORNING
+2FA1 ; 8FB5 # KANGXI RADICAL WALK
+2FA2 ; 9091 # KANGXI RADICAL CITY
+2FA3 ; 9149 # KANGXI RADICAL WINE
+2FA4 ; 91C6 # KANGXI RADICAL DISTINGUISH
+2FA5 ; 91CC # KANGXI RADICAL VILLAGE
+2FA6 ; 91D1 # KANGXI RADICAL GOLD
+2FA7 ; 9577 # KANGXI RADICAL LONG
+2FA8 ; 9580 # KANGXI RADICAL GATE
+2FA9 ; 961C # KANGXI RADICAL MOUND
+2FAA ; 96B6 # KANGXI RADICAL SLAVE
+2FAB ; 96B9 # KANGXI RADICAL SHORT TAILED BIRD
+2FAC ; 96E8 # KANGXI RADICAL RAIN
+2FAD ; 9751 # KANGXI RADICAL BLUE
+2FAE ; 975E # KANGXI RADICAL WRONG
+2FAF ; 9762 # KANGXI RADICAL FACE
+2FB0 ; 9769 # KANGXI RADICAL LEATHER
+2FB1 ; 97CB # KANGXI RADICAL TANNED LEATHER
+2FB2 ; 97ED # KANGXI RADICAL LEEK
+2FB3 ; 97F3 # KANGXI RADICAL SOUND
+2FB4 ; 9801 # KANGXI RADICAL LEAF
+2FB5 ; 98A8 # KANGXI RADICAL WIND
+2FB6 ; 98DB # KANGXI RADICAL FLY
+2FB7 ; 98DF # KANGXI RADICAL EAT
+2FB8 ; 9996 # KANGXI RADICAL HEAD
+2FB9 ; 9999 # KANGXI RADICAL FRAGRANT
+2FBA ; 99AC # KANGXI RADICAL HORSE
+2FBB ; 9AA8 # KANGXI RADICAL BONE
+2FBC ; 9AD8 # KANGXI RADICAL TALL
+2FBD ; 9ADF # KANGXI RADICAL HAIR
+2FBE ; 9B25 # KANGXI RADICAL FIGHT
+2FBF ; 9B2F # KANGXI RADICAL SACRIFICIAL WINE
+2FC0 ; 9B32 # KANGXI RADICAL CAULDRON
+2FC1 ; 9B3C # KANGXI RADICAL GHOST
+2FC2 ; 9B5A # KANGXI RADICAL FISH
+2FC3 ; 9CE5 # KANGXI RADICAL BIRD
+2FC4 ; 9E75 # KANGXI RADICAL SALT
+2FC5 ; 9E7F # KANGXI RADICAL DEER
+2FC6 ; 9EA5 # KANGXI RADICAL WHEAT
+2FC7 ; 9EBB # KANGXI RADICAL HEMP
+2FC8 ; 9EC3 # KANGXI RADICAL YELLOW
+2FC9 ; 9ECD # KANGXI RADICAL MILLET
+2FCA ; 9ED1 # KANGXI RADICAL BLACK
+2FCB ; 9EF9 # KANGXI RADICAL EMBROIDERY
+2FCC ; 9EFD # KANGXI RADICAL FROG
+2FCD ; 9F0E # KANGXI RADICAL TRIPOD
+2FCE ; 9F13 # KANGXI RADICAL DRUM
+2FCF ; 9F20 # KANGXI RADICAL RAT
+2FD0 ; 9F3B # KANGXI RADICAL NOSE
+2FD1 ; 9F4A # KANGXI RADICAL EVEN
+2FD2 ; 9F52 # KANGXI RADICAL TOOTH
+2FD3 ; 9F8D # KANGXI RADICAL DRAGON
+2FD4 ; 9F9C # KANGXI RADICAL TURTLE
+2FD5 ; 9FA0 # KANGXI RADICAL FLUTE
+31C6 ; 200CC # CJK STROKE HZG
+31CF ; 4E40 # CJK STROKE N
+31D0 ; 4E00 # CJK STROKE H
+31D1 ; 4E28 # CJK STROKE S
+31D2..31D3 ; 4E3F # [2] CJK STROKE P..CJK STROKE SP
+31D4 ; 4E36 # CJK STROKE D
+31D5 ; 200CD # CJK STROKE HZ
+31D6 ; 4E5B # CJK STROKE HG
+31D7 ; 200CA # CJK STROKE SZ
+31D8 ; 200CE # CJK STROKE SWZ
+31D9 ; 2010C # CJK STROKE ST
+31DA ; 4E85 # CJK STROKE SG
+31DB ; 21FE8 # CJK STROKE PD
+31DC ; 200CB # CJK STROKE PZ
+31DD ; 4E40 # CJK STROKE TN
+31DE ; 200D1 # CJK STROKE SZZ
+31DF ; 4E5A # CJK STROKE SWG
+31E0 ; 4E59 # CJK STROKE HXWG
+31E1 ; 2010E # CJK STROKE HZZZG
+
+# The following characters have no equivalent CJK unified ideograph.
+
+# 2E80; CJK RADICAL REPEAT
+# 31C0; CJK STROKE T
+# 31C1; CJK STROKE WG
+# 31C2; CJK STROKE XG
+# 31C3; CJK STROKE BXG
+# 31C4; CJK STROKE SW
+# 31C5; CJK STROKE HZZ
+# 31C7; CJK STROKE HP
+# 31C8; CJK STROKE HZWG
+# 31C9; CJK STROKE SZWG
+# 31CA; CJK STROKE HZT
+# 31CB; CJK STROKE HZZP
+# 31CC; CJK STROKE HPWG
+# 31CD; CJK STROKE HZW
+# 31CE; CJK STROKE HZZZ
+# 31E2; CJK STROKE PG
+# 31E3; CJK STROKE Q
+# 31E4; CJK STROKE HXG
+# 31E5; CJK STROKE SZP
+
+# EOF
diff --git a/crates/utopia-ingest/src/lib.rs b/crates/utopia-ingest/src/lib.rs
index 94a76e44..df77ff82 100644
--- a/crates/utopia-ingest/src/lib.rs
+++ b/crates/utopia-ingest/src/lib.rs
@@ -164,6 +164,7 @@ mod reader_tests {
mod blocks;
mod chunker;
pub mod html;
+mod lookalikes;
pub mod mineru;
pub mod ontology_rdf;
mod parsers;
diff --git a/crates/utopia-ingest/src/lookalikes.rs b/crates/utopia-ingest/src/lookalikes.rs
new file mode 100644
index 00000000..ca168be1
--- /dev/null
+++ b/crates/utopia-ingest/src/lookalikes.rs
@@ -0,0 +1,123 @@
+//! PDF 文字层里长得一样、码位不同的字:部首与连字换回它们显示的那个字。
+//!
+//! PDF 画的是字形,文字层靠字体的 ToUnicode 表把字形指回码位。同一个字形在字体里常常
+//! 既是一个部首又是一个统一汉字——苹方里「⼀」(U+2F00,康熙部首)和「一」(U+4E00)是
+//! 同一个字形——ToUnicode 指到部首码位,读出来的字看着一样,码位却是部首:搜「日期」
+//! 找不到「⽇期」,「王⼤⼒」和「王大力」是两个名字。macOS 生成的 PDF(Quartz、Chrome,
+//! 用苹方)里,有部首孪生码位的汉字大都这样读出来。拉丁连字同理:「fiscal」的 fi 是一个
+//! 字形、一个码位(U+FB01)。
+//!
+//! 部首换成 Unicode 说它等同的那个统一汉字:UCD 的 `EquivalentUnifiedIdeograph.txt`,
+//! 原样收在旁边。康熙部首 NFKC 也换得了,部首补充区(⻄、⻓)NFKC 不管,所以用这张表。
+//! 表里的笔画不换:文档里写笔画,是当笔画用的。连字拆成字母。别的字一概不动——这不是
+//! NFKC,全角标点、全角数字照旧。
+
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::sync::OnceLock;
+
+/// Unicode 18.0.0 的 EquivalentUnifiedIdeograph.txt,原样收录
+const EQUIVALENTS: &str = include_str!("EquivalentUnifiedIdeograph.txt");
+
+/// 两个部首区:部首补充(U+2E80–2EFF)与康熙部首(U+2F00–2FDF)
+fn is_radical(c: char) -> bool {
+ matches!(c, '\u{2E80}'..='\u{2EFF}' | '\u{2F00}'..='\u{2FDF}')
+}
+
+/// 部首 → 它等同的统一汉字。表里没有的部首(⺀)不在这里
+fn radicals() -> &'static HashMap<char, char> {
+ static MAP: OnceLock<HashMap<char, char>> = OnceLock::new();
+ MAP.get_or_init(|| {
+ let hex = |s: &str| u32::from_str_radix(s.trim(), 16).ok();
+ let mut map = HashMap::new();
+ // 一行一条:`2F08 ; 4EBA # KANGXI RADICAL MAN`,左边也可以是区间 `2E8C..2E8D`
+ for line in EQUIVALENTS.lines() {
+ let data = line.split('#').next().unwrap_or_default();
+ let Some((from, to)) = data.split_once(';') else {
+ continue;
+ };
+ let (lo, hi) = from.split_once("..").unwrap_or((from, from));
+ let (Some(lo), Some(hi), Some(to)) =
+ (hex(lo), hex(hi), hex(to).and_then(char::from_u32))
+ else {
+ continue;
+ };
+ for c in (lo..=hi)
+ .filter_map(char::from_u32)
+ .filter(|&c| is_radical(c))
+ {
+ map.insert(c, to);
+ }
+ }
+ map
+ })
+}
+
+/// 拉丁连字 → 字母(U+FB00–FB06,与 NFKC 拆得一样)
+fn ligature(c: char) -> Option<&'static str> {
+ Some(match c {
+ '\u{FB00}' => "ff",
+ '\u{FB01}' => "fi",
+ '\u{FB02}' => "fl",
+ '\u{FB03}' => "ffi",
+ '\u{FB04}' => "ffl",
+ '\u{FB05}' | '\u{FB06}' => "st",
+ _ => return None,
+ })
+}
+
+/// 部首、连字换回它们显示的那个字;一个都没有时原样借用
+pub(crate) fn plain(text: &str) -> Cow<'_, str> {
+ if !text.chars().any(|c| is_radical(c) || ligature(c).is_some()) {
+ return Cow::Borrowed(text);
+ }
+ let mut out = String::with_capacity(text.len());
+ for c in text.chars() {
+ match ligature(c) {
+ Some(letters) => out.push_str(letters),
+ None if is_radical(c) => out.push(radicals().get(&c).copied().unwrap_or(c)),
+ None => out.push(c),
+ }
+ }
+ Cow::Owned(out)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn every_kangxi_radical_reads_as_the_ideograph_unicode_names() {
+ let kangxi: Vec<char> = ('\u{2F00}'..='\u{2FD5}').collect();
+ assert_eq!(kangxi.len(), 214);
+ for c in kangxi {
+ let ideograph = radicals().get(&c).copied();
+ assert!(
+ ideograph.is_some_and(|i| !is_radical(i)),
+ "{c} (U+{:04X}) → {ideograph:?}",
+ c as u32
+ );
+ }
+ // 补充区也在表里,除了 Unicode 说没有等同字的 ⺀
+ assert_eq!(radicals().len(), 214 + 114);
+ assert_eq!(radicals().get(&'\u{2E80}'), None);
+ }
+
+ #[test]
+ fn the_lookalikes_of_a_mac_pdf_read_as_the_characters_they_show() {
+ let read = "项⽬负责⼈为王⼤⼒,现⾦流量净额同⽐增⻓。⼀⾏⻄⺟⻳⺅ fiscal flow ffl";
+ assert_eq!(
+ plain(read),
+ "项目负责人为王大力,现金流量净额同比增长。一行西母龟亻 fiscal flow ffl"
+ );
+ // 没有等同字的部首照旧
+ assert_eq!(plain("⺀"), "⺀");
+ }
+
+ #[test]
+ fn other_text_is_borrowed_as_it_is() {
+ // 全角标点、全角数字、全角字母不动:这不是 NFKC
+ let text = "营业收入125,430元(同比增长13.77%);ABC “引号” café";
+ assert!(matches!(plain(text), Cow::Borrowed(t) if t == text));
+ }
+}
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index 1058a9b8..62b33720 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -48,7 +48,17 @@ fn draws_text(bytes: &[u8]) -> bool {
})
}
-/// PDF 的文字层。取不出来时交给外面的 `pdftotext`——它是另一个进程,所以那边再崩也带不走
+/// PDF 的文字层。字体的 ToUnicode 常把字形指到长得一样的部首、连字码位,`pdf_extract`
+/// 读出来的照收:换回它们显示的那个字([`crate::lookalikes`])
+pub fn pdf(bytes: &[u8]) -> anyhow::Result<String> {
+ let text = text_layer(bytes)?;
+ Ok(match crate::lookalikes::plain(&text) {
+ std::borrow::Cow::Borrowed(_) => text,
+ std::borrow::Cow::Owned(plain) => plain,
+ })
+}
+
+/// 取文字层。取不出来时交给外面的 `pdftotext`——它是另一个进程,所以那边再崩也带不走
/// 一个工作线程。
///
/// 回退也空手而归时,[`draws_text`] 决定该说哪句话:这份文件没画过字,那是扫描件,交给
@@ -56,7 +66,7 @@ fn draws_text(bytes: &[u8]) -> bool {
/// 的原话往外抛。报错的那句话值得较真:说成「没有文字层」会把人支去配一个 OCR 服务,而
/// 这份文件的文字层好端端地在那儿(#739:同名的 `pdftotext` 有两个实现,Xpdf 那个和缺了
/// CJK CMap 数据的 Poppler 都会静静地返回空)
-pub fn pdf(bytes: &[u8]) -> anyhow::Result<String> {
+fn text_layer(bytes: &[u8]) -> anyhow::Result<String> {
let extracted = std::panic::catch_unwind(|| pdf_extract::extract_text_from_mem(bytes));
let original_error = match extracted {
Ok(Ok(text)) if !text.trim().is_empty() => return Ok(text),
diff --git a/crates/utopia-ingest/tests/pdf_lookalikes.rs b/crates/utopia-ingest/tests/pdf_lookalikes.rs
new file mode 100644
index 00000000..e3ebab19
--- /dev/null
+++ b/crates/utopia-ingest/tests/pdf_lookalikes.rs
@@ -0,0 +1,128 @@
+//! PDF 文字层里长得一样的码位(部首、连字)读成它们显示的那个字。
+//!
+//! 照 macOS 生成的 PDF 的样子拼一份:Type0 字体、Identity-H,ToUnicode 把字形指到部首和
+//! 连字的码位——Quartz 与 Chrome 用苹方生成的 PDF 就是这样指的。不带字体文件,取字只看
+//! ToUnicode。
+
+use lopdf::content::{Content, Operation};
+use lopdf::{dictionary, Document, Object, Stream, StringFormat};
+
+// Build a one-line PDF whose glyph i+1 maps to the i-th character of `shown` through ToUnicode.
+fn pdf_showing(shown: &str) -> Vec<u8> {
+ let chars: Vec<char> = shown.chars().collect();
+ let mut cmap = String::from(
+ "/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n\
+ /CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n\
+ /CMapName /Adobe-Identity-UCS def\n/CMapType 2 def\n\
+ 1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n",
+ );
+ cmap.push_str(&format!("{} beginbfchar\n", chars.len()));
+ for (i, c) in chars.iter().enumerate() {
+ let utf16: String = c
+ .encode_utf16(&mut [0; 2])
+ .iter()
+ .map(|unit| format!("{unit:04X}"))
+ .collect();
+ cmap.push_str(&format!("<{:04X}> <{utf16}>\n", i + 1));
+ }
+ cmap.push_str("endbfchar\nendcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n");
+
+ let mut document = Document::with_version("1.4");
+ let pages_id = document.new_object_id();
+ let to_unicode_id = document.add_object(Stream::new(dictionary! {}, cmap.into_bytes()));
+ let descriptor_id = document.add_object(dictionary! {
+ "Type" => "FontDescriptor",
+ "FontName" => "Lookalikes-Regular",
+ "Flags" => 4,
+ "FontBBox" => vec![0.into(), (-200).into(), 1000.into(), 900.into()],
+ "ItalicAngle" => 0,
+ "Ascent" => 880,
+ "Descent" => -120,
+ "CapHeight" => 880,
+ "StemV" => 80,
+ });
+ let cid_font_id = document.add_object(dictionary! {
+ "Type" => "Font",
+ "Subtype" => "CIDFontType2",
+ "BaseFont" => "Lookalikes-Regular",
+ "CIDSystemInfo" => dictionary! {
+ "Registry" => Object::string_literal("Adobe"),
+ "Ordering" => Object::string_literal("Identity"),
+ "Supplement" => 0,
+ },
+ "FontDescriptor" => descriptor_id,
+ "DW" => 1000,
+ });
+ let font_id = document.add_object(dictionary! {
+ "Type" => "Font",
+ "Subtype" => "Type0",
+ "BaseFont" => "Lookalikes-Regular",
+ "Encoding" => "Identity-H",
+ "DescendantFonts" => vec![Object::Reference(cid_font_id)],
+ "ToUnicode" => to_unicode_id,
+ });
+ let resources_id = document.add_object(dictionary! {
+ "Font" => dictionary! { "F1" => font_id },
+ });
+ let glyphs: Vec<u8> = (1..=chars.len() as u16)
+ .flat_map(u16::to_be_bytes)
+ .collect();
+ let content = Content {
+ operations: vec![
+ Operation::new("BT", vec![]),
+ Operation::new("Tf", vec!["F1".into(), 12.into()]),
+ Operation::new("Td", vec![50.into(), 750.into()]),
+ Operation::new(
+ "Tj",
+ vec![Object::String(glyphs, StringFormat::Hexadecimal)],
+ ),
+ Operation::new("ET", vec![]),
+ ],
+ };
+ let content_id = document.add_object(Stream::new(dictionary! {}, content.encode().unwrap()));
+ let page_id = document.add_object(dictionary! {
+ "Type" => "Page",
+ "Parent" => pages_id,
+ "Contents" => content_id,
+ });
+ document.objects.insert(
+ pages_id,
+ Object::Dictionary(dictionary! {
+ "Type" => "Pages",
+ "Kids" => vec![Object::Reference(page_id)],
+ "Count" => 1,
+ "Resources" => resources_id,
+ "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
+ }),
+ );
+ let catalog_id = document.add_object(dictionary! {
+ "Type" => "Catalog",
+ "Pages" => pages_id,
+ });
+ document.trailer.set("Root", catalog_id);
+ let mut bytes = Vec::new();
+ document.save_to(&mut bytes).unwrap();
+ bytes
+}
+
+#[test]
+fn a_mac_pdf_is_read_with_the_characters_it_shows() {
+ // ⽂ ⾃ ⽇ ⾏ ⼀ ⽤ 是康熙部首,⻄ 是部首补充区的,fi fl 是连字
+ let shown = "时间线按⽂件⾃带的⽇期排⾏,⼀律⽤⻄历。fiscal flow";
+ let parsed = utopia_ingest::parse("report.pdf", &pdf_showing(shown)).unwrap();
+ assert!(
+ parsed
+ .text
+ .contains("时间线按文件自带的日期排行,一律用西历。fiscal flow"),
+ "{}",
+ parsed.text
+ );
+ assert!(
+ !parsed.text.chars().any(|c| matches!(
+ c,
+ '\u{2E80}'..='\u{2EFF}' | '\u{2F00}'..='\u{2FDF}' | '\u{FB00}'..='\u{FB06}'
+ )),
+ "{}",
+ parsed.text
+ );
+}