Skip to content

The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.

fix(db): Read MySQL catalog names as text, not hex (#212)

l0ng-ai6767a5ff8c2e
diff --git a/crates/ide-db/src/conn.rs b/crates/ide-db/src/conn.rs
index 85eb6806b..93c8ee6ba 100644
--- a/crates/ide-db/src/conn.rs
+++ b/crates/ide-db/src/conn.rs
@@ -88,10 +88,46 @@ fn hex(bytes: &[u8]) -> String {
s
}
+/// How [`cell`] shows a value whose bytes are not text.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum Cells {
+ /// Result grids: binary values as `0x…` hex.
+ Display,
+ /// Catalog queries: names and definitions, so bytes are read as UTF-8 whatever the
+ /// column's type (MySQL flags `utf8mb3_bin` catalog columns BINARY, and sends every
+ /// column as binary under `character_set_results = binary`).
+ Text,
+}
+
+/// Whether a value of type `ty` is binary data (see [`binary_type`]).
+fn is_binary<DB: sqlx::Database>(ty: &DB::TypeInfo) -> bool
+where
+ String: sqlx::Type<DB>,
+{
+ binary_type(ty.name(), <String as sqlx::Type<DB>>::compatible(ty))
+}
+
+/// A BLOB / BINARY type that the driver does not read as text: MySQL flags text in a
+/// binary collation (`utf8mb4_bin`) BINARY too, and only the `binary` collation is bytes.
+fn binary_type(name: &str, text: bool) -> bool {
+ let name = name.to_ascii_uppercase();
+ (name.contains("BLOB") || name.contains("BINARY")) && !text
+}
+
+/// Bytes as text: UTF-8, else hex (grids) or with invalid sequences replaced (catalog names).
+fn bytes_text(bytes: Vec<u8>, mode: Cells) -> String {
+ match (String::from_utf8(bytes), mode) {
+ (Ok(s), _) => s,
+ (Err(e), Cells::Display) => hex(e.as_bytes()),
+ (Err(e), Cells::Text) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
+ }
+}
+
/// The text of column `i` of `row`.
-fn cell<R: Row>(row: &R, i: usize) -> Option<String>
+fn cell<R: Row>(row: &R, i: usize, mode: Cells) -> Option<String>
where
usize: ColumnIndex<R>,
+ String: sqlx::Type<R::Database>,
for<'r> String: Decode<'r, R::Database>,
for<'r> Vec<u8>: Decode<'r, R::Database>,
{
@@ -99,18 +135,14 @@ where
if raw.is_null() {
return None;
}
- let ty = raw.type_info().name().to_ascii_uppercase();
- if ty.contains("BLOB") || ty.contains("BINARY") {
- let raw = row.try_get_raw(i).ok()?;
+ if mode == Cells::Display && is_binary::<R::Database>(&raw.type_info()) {
return <Vec<u8> as Decode<R::Database>>::decode(raw).ok().map(|b| hex(&b));
}
match <String as Decode<R::Database>>::decode(raw) {
Ok(s) => Some(s),
Err(_) => {
let raw = row.try_get_raw(i).ok()?;
- <Vec<u8> as Decode<R::Database>>::decode(raw)
- .ok()
- .map(|b| String::from_utf8(b).unwrap_or_else(|e| hex(e.as_bytes())))
+ <Vec<u8> as Decode<R::Database>>::decode(raw).ok().map(|b| bytes_text(b, mode))
}
}
}
@@ -122,9 +154,10 @@ fn result_columns<R: Row>(row: &R) -> Vec<ResultColumn> {
.collect()
}
-/// Streams the results of `$sql` on executor `$exec`, keeping at most `$max` rows.
+/// Streams the results of `$sql` on executor `$exec`, keeping at most `$max` rows
+/// with their cells read as `$mode`.
macro_rules! fetch {
- ($exec:expr, $sql:expr, $max:expr) => {{
+ ($exec:expr, $sql:expr, $max:expr, $mode:expr) => {{
use futures::StreamExt;
let mut stream = sqlx::raw_sql(sqlx::AssertSqlSafe($sql.to_owned())).fetch_many($exec);
let mut columns: Vec<ResultColumn> = Vec::new();
@@ -145,7 +178,7 @@ macro_rules! fetch {
has_more = true;
break;
}
- rows.push((0..row.len()).map(|i| cell(&row, i)).collect());
+ rows.push((0..row.len()).map(|i| cell(&row, i, $mode)).collect());
}
Err(e) => {
failure = Some(db_err(e));
@@ -167,7 +200,7 @@ type Fetched = (Vec<ResultColumn>, Vec<Vec<Option<String>>>, Option<u64>, bool,
/// The server-side id of a connection (to cancel its statement from another one).
macro_rules! backend_id {
($conn:expr, $sql:expr) => {{
- let fetched: IdeResult<Fetched> = fetch!($conn, $sql, 1);
+ let fetched: IdeResult<Fetched> = fetch!($conn, $sql, 1, Cells::Text);
fetched
.ok()
.and_then(|f| f.1.into_iter().next())
@@ -431,9 +464,9 @@ impl Pool {
with_pool!(self, p => p.is_closed())
}
- /// Runs `sql` (no row limit beyond `max`), for introspection.
+ /// Runs `sql` (no row limit beyond `max`), for introspection: cells are text (see [`Cells::Text`]).
pub async fn rows(&self, sql: &str, max: usize) -> IdeResult<Vec<Vec<Option<String>>>> {
- let fetched: IdeResult<Fetched> = with_pool!(self, p => fetch!(p, sql, max));
+ let fetched: IdeResult<Fetched> = with_pool!(self, p => fetch!(p, sql, max, Cells::Text));
Ok(fetched?.1)
}
@@ -530,7 +563,7 @@ impl Held {
None => None,
};
let r = tokio::select! {
- r = async { fetch!(&mut **$conn, sql, max_rows) } => r,
+ r = async { fetch!(&mut **$conn, sql, max_rows, Cells::Display) } => r,
() = cancel.cancelled() => Err(IdeError::cancelled()),
};
if cancel.is_cancelled() {
@@ -649,4 +682,35 @@ mod tests {
let s = DataSource { properties: vec![prop("cache_size", "-2000")], ..Default::default() };
assert_eq!(Setup::of(&s).unwrap().params, [("cache_size".to_owned(), "-2000".to_owned())]);
}
+
+ #[test]
+ fn only_bytes_typed_values_are_binary() {
+ // (type name as the driver reports it, the driver reads it as text, binary)
+ let cases = [
+ ("VARBINARY", false, true),
+ ("BLOB", false, true),
+ ("LONGBLOB", false, true),
+ ("BINARY", false, true),
+ // MySQL catalog columns (`utf8mb3_bin`) and `utf8mb4_bin` text: flagged BINARY.
+ ("VARBINARY", true, false),
+ ("BLOB", true, false),
+ ("VARCHAR", true, false),
+ ("TEXT", true, false),
+ ("BYTEA", false, false),
+ ("INT", false, false),
+ ];
+ for (name, text, binary) in cases {
+ assert_eq!(binary_type(name, text), binary, "{name} text={text}");
+ }
+ }
+
+ #[test]
+ fn catalog_bytes_are_names_and_grid_bytes_are_hex() {
+ // `oi_数据` as MySQL sends it under `character_set_results = binary`.
+ let name = "oi_数据".as_bytes().to_vec();
+ assert_eq!(bytes_text(name.clone(), Cells::Text), "oi_数据");
+ assert_eq!(bytes_text(name, Cells::Display), "oi_数据");
+ assert_eq!(bytes_text(vec![0xCA, 0xFE], Cells::Display), "0xCAFE");
+ assert_eq!(bytes_text(vec![b'a', 0xFF], Cells::Text), "a\u{FFFD}");
+ }
}
diff --git a/crates/ide-db/tests/mysql.rs b/crates/ide-db/tests/mysql.rs
new file mode 100644
index 000000000..a6ab2431a
--- /dev/null
+++ b/crates/ide-db/tests/mysql.rs
@@ -0,0 +1,87 @@
+//! Introspection against a real MySQL / MariaDB server, skipped unless
+//! `OPENIDEA_TEST_MYSQL=host:port:user:password` names a throwaway server.
+
+use ide_core::{CancellationToken, IdeResult};
+use ide_db::{DataSource, DbDriver, DbManager, DbOptions};
+
+const SCHEMAS: [&str; 2] = ["oi_bin_meta", "oi_数据"];
+
+fn server() -> Option<(String, u16, String, String)> {
+ let v = std::env::var("OPENIDEA_TEST_MYSQL").ok()?;
+ let mut it = v.splitn(4, ':');
+ Some((it.next()?.into(), it.next()?.parse().ok()?, it.next()?.into(), it.next().unwrap_or("").into()))
+}
+
+async fn manager(tmp: &tempfile::TempDir, startup_script: &str) -> Option<DbManager> {
+ let (host, port, user, password) = server()?;
+ let root = tmp.path().to_path_buf();
+ let ssh = std::sync::Arc::new(ide_ssh::SshManager::new(root.join("ssh-passwords.json")));
+ let db = DbManager::new(root.clone(), root.join("passwords.json"), root.join("history.json"), ssh);
+ db.save_sources(vec![DataSource {
+ id: "m".into(),
+ name: "mysql".into(),
+ driver: DbDriver::Mysql,
+ host,
+ port: Some(port),
+ user,
+ options: DbOptions { startup_script: startup_script.into(), ..Default::default() },
+ ..Default::default()
+ }])
+ .await
+ .unwrap();
+ db.set_password("m", Some(password), false).await.unwrap();
+ Some(db)
+}
+
+async fn create_schemas(db: &DbManager) {
+ let c = CancellationToken::new();
+ for s in SCHEMAS {
+ for stmt in [
+ format!("DROP DATABASE IF EXISTS `{s}`"),
+ format!("CREATE DATABASE `{s}` CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"),
+ format!(
+ "CREATE TABLE `{s}`.`用户` (id INT PRIMARY KEY, name VARCHAR(20) COLLATE utf8mb4_bin, raw VARBINARY(4))"
+ ),
+ format!("CREATE INDEX `名字` ON `{s}`.`用户` (name)"),
+ format!("INSERT INTO `{s}`.`用户` VALUES (1, 'ann', 0xCAFE)"),
+ ] {
+ db.execute("m", &stmt, 10, &c).await.unwrap();
+ }
+ }
+}
+
+async fn assert_metadata_is_text(db: &DbManager) -> IdeResult<()> {
+ let schemas = db.schemas("m").await?;
+ for s in SCHEMAS {
+ assert!(schemas.iter().any(|n| n == s), "{s} missing from {schemas:?}");
+ let tables = db.tables("m", s).await?;
+ assert_eq!(tables.iter().map(|t| t.name.as_str()).collect::<Vec<_>>(), ["用户"], "tables of {s}");
+ let d = db.table_details("m", s, "用户").await?;
+ let cols: Vec<_> = d.columns.iter().map(|c| c.name.as_str()).collect();
+ assert_eq!(cols, ["id", "name", "raw"]);
+ assert!(d.indexes.iter().any(|i| i.name == "名字"), "{:?}", d.indexes);
+ assert!(db.ddl("m", s, "用户").await?.contains("CREATE TABLE `用户`"));
+ }
+ let completion = db.completion("m").await?;
+ let json = serde_json::to_string(&completion).unwrap();
+ assert!(json.contains("oi_数据") && json.contains("用户"), "{json}");
+ Ok(())
+}
+
+// One test: both phases share the schemas they create.
+#[tokio::test]
+async fn metadata_names_are_text_whatever_the_column_collation() -> IdeResult<()> {
+ let tmp = tempfile::tempdir().unwrap();
+ let Some(db) = manager(&tmp, "").await else { return Ok(()) };
+ create_schemas(&db).await;
+ // MySQL 8 sends `utf8mb3_bin` catalog names flagged BINARY (lower_case_table_names = 0).
+ assert_metadata_is_text(&db).await?;
+ // `_bin` text is text in the grid; true binary stays hex.
+ let r = db.execute("m", "SELECT name, raw FROM `oi_数据`.`用户`", 10, &CancellationToken::new()).await?;
+ assert_eq!(r.rows, vec![vec![Some("ann".to_owned()), Some("0xCAFE".to_owned())]]);
+
+ // Every text column arrives with the `binary` collation.
+ let tmp = tempfile::tempdir().unwrap();
+ let raw = manager(&tmp, "SET character_set_results = binary").await.unwrap();
+ assert_metadata_is_text(&raw).await
+}