Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

8.8 KB
Raw Blame
import type { Issue, Operator } from './parser.types';
import { ERROR_CODES } from './errors';
import { isWhitespace } from './string';

export type Token =
  | { type: 'LPAREN' }
  | { type: 'RPAREN' }
  | { type: 'AND' }
  | { type: 'OR' }
  | { type: 'NOT' }
  | { type: 'FILTER'; field: string; operator: Operator; value: string }
  | { type: 'TEXT'; value: string }
  | { type: 'EOF' };

export type TokenizeResult = {
  tokens: Token[];
  issues: Issue[];
};

// Unified escape handling utilities
const unescapeBackslashes = (str: string): string => str.replace(/\\(.)/g, '$1');
const unescapeColons = (str: string): string => str.replace(/\\:/g, ':');

type StopCondition = (char: string) => boolean;

function isWhitespaceOrParen(char: string): boolean {
  return isWhitespace(char) || char === '(' || char === ')';
}

export function tokenize({
  query,
  maxTokens,
}: {
  query: string;
  maxTokens: number;
}): TokenizeResult {
  const tokens: Token[] = [];
  const issues: Issue[] = [];
  let pos = 0;

  const peek = (): string | undefined => query[pos];
  const advance = (): string => query[pos++] || '';

  const skipWhitespace = (): void => {
    while (pos < query.length) {
      const char = query[pos];
      if (!char || !isWhitespace(char)) {
        break;
      }
      pos++;
    }
  };

  const readQuotedString = (): string | undefined => {
    if (peek() !== '"') {
      return undefined;
    }

    advance(); // Skip opening quote
    let value = '';
    let escaped = false;

    while (pos < query.length) {
      const char = peek();

      if (!char) {
        break;
      }

      if (escaped) {
        value += char;
        escaped = false;
        advance();
      } else if (char === '\\') {
        escaped = true;
        advance();
      } else if (char === '"') {
        advance(); // Skip closing quote
        return value;
      } else {
        value += char;
        advance();
      }
    }

    issues.push({
      code: ERROR_CODES.UNCLOSED_QUOTED_STRING,
      message: 'Unclosed quoted string',
    });
    return value;
  };

  // Unified function to read unquoted values with configurable stop conditions and escape handling
  const readUnquotedValue = ({
    stopCondition,
    processEscapes,
    stopAtQuote = false,
  }: {
    stopCondition: StopCondition;
    processEscapes: boolean;
    stopAtQuote?: boolean;
  }): string => {
    let value = '';

    while (pos < query.length) {
      const char = peek();

      if (!char || stopCondition(char)) {
        break;
      }

      if (stopAtQuote && char === '"') {
        break;
      }

      if (processEscapes && char === '\\') {
        advance();
        if (pos < query.length) {
          value += advance();
        }
      } else {
        value += advance();
      }
    }

    return value;
  };

  const readUnquotedToken = (): string => {
    // Keep backslashes in unquoted tokens so parseFilter can detect escaped colons
    return readUnquotedValue({
      stopCondition: isWhitespaceOrParen,
      processEscapes: false,
      stopAtQuote: true,
    });
  };

  const readFilterValue = (): string => {
    skipWhitespace();

    // Check if value is quoted
    const quotedValue = readQuotedString();
    if (quotedValue !== undefined) {
      return quotedValue;
    }

    // Read unquoted value with escape processing, then unescape colons
    const value = readUnquotedValue({
      stopCondition: isWhitespaceOrParen,
      processEscapes: true,
    });

    return unescapeColons(value);
  };

  const readFilterOperatorAndValue = (): { operator: Operator; value: string } => {
    let operator: Operator = '=';

    if (query.startsWith('>=', pos)) {
      operator = '>=';
      pos += 2;
    } else if (query.startsWith('<=', pos)) {
      operator = '<=';
      pos += 2;
    } else if (peek() === '>') {
      operator = '>';
      pos++;
    } else if (peek() === '<') {
      operator = '<';
      pos++;
    } else if (peek() === '=') {
      operator = '=';
      pos++;
    }

    const value = readFilterValue();

    return { operator, value };
  };

  const hasUnescapedColon = (str: string): boolean => {
    for (let i = 0; i < str.length; i++) {
      if (str[i] === ':' && (i === 0 || str[i - 1] !== '\\')) {
        return true;
      }
    }
    return false;
  };

  const parseFilter = (token: string): Token | undefined => {
    // Check for unescaped colons
    if (!hasUnescapedColon(token)) {
      return undefined;
    }

    // Find first unescaped colon
    let firstColonIndex = -1;
    for (let i = 0; i < token.length; i++) {
      if (token[i] === ':' && (i === 0 || token[i - 1] !== '\\')) {
        firstColonIndex = i;
        break;
      }
    }

    if (firstColonIndex === -1) {
      return undefined;
    }

    const field = unescapeColons(token.slice(0, firstColonIndex));
    const afterColon = token.slice(firstColonIndex + 1);

    // Check for operator at the start
    let operator: Operator = '=';
    let operatorLength = 0;

    if (afterColon.startsWith('>=')) {
      operator = '>=';
      operatorLength = 2;
    } else if (afterColon.startsWith('<=')) {
      operator = '<=';
      operatorLength = 2;
    } else if (afterColon.startsWith('>')) {
      operator = '>';
      operatorLength = 1;
    } else if (afterColon.startsWith('<')) {
      operator = '<';
      operatorLength = 1;
    } else if (afterColon.startsWith('=')) {
      operator = '=';
      operatorLength = 1;
    }

    // If there's nothing after the operator in the token, read the value from input
    let value = unescapeColons(afterColon.slice(operatorLength));

    // If the value is empty and we still have input, this might be a case like "tag:" followed by a quoted string
    if (!value) {
      value = readFilterValue();
    }

    return { type: 'FILTER', field, operator, value };
  };

  while (pos < query.length) {
    if (tokens.length >= maxTokens) {
      issues.push({
        code: ERROR_CODES.MAX_TOKENS_EXCEEDED,
        message: `Maximum token limit of ${maxTokens} exceeded`,
      });
      break;
    }

    skipWhitespace();

    if (pos >= query.length) {
      break;
    }

    const char = peek();

    // Handle parentheses
    if (char === '(') {
      advance();
      tokens.push({ type: 'LPAREN' });
      continue;
    }

    if (char === ')') {
      advance();
      tokens.push({ type: 'RPAREN' });
      continue;
    }

    // Handle negation prefix
    const nextChar = query[pos + 1];
    if (char === '-' && nextChar && !isWhitespace(nextChar)) {
      advance();
      skipWhitespace();

      // Read the next token (could be quoted or unquoted)
      const quotedValue = readQuotedString();

      // If quoted, check if it's a quoted field name (followed by ':')
      if (quotedValue !== undefined) {
        if (peek() === ':') {
          advance(); // Consume ':'
          const { operator, value } = readFilterOperatorAndValue();
          tokens.push({ type: 'NOT' });
          tokens.push({ type: 'FILTER', field: quotedValue, operator, value });
        } else {
          tokens.push({ type: 'NOT' });
          tokens.push({ type: 'TEXT', value: quotedValue });
        }
        continue;
      }

      const token = readUnquotedToken();

      // Try to parse as filter
      const filter = parseFilter(token);
      if (filter) {
        tokens.push({ type: 'NOT' });
        tokens.push(filter);
      } else {
        // If not a filter, treat as negated text search
        tokens.push({ type: 'NOT' });
        tokens.push({ type: 'TEXT', value: unescapeBackslashes(token) });
      }
      continue;
    }

    // Read quoted or unquoted token
    const quotedValue = readQuotedString();

    // If quoted, always treat as TEXT or FILTER — never as a keyword operator
    if (quotedValue !== undefined) {
      if (peek() === ':') {
        advance(); // Consume ':'
        const { operator, value } = readFilterOperatorAndValue();
        tokens.push({ type: 'FILTER', field: quotedValue, operator, value });
      } else {
        tokens.push({ type: 'TEXT', value: quotedValue });
      }
      continue;
    }

    const token = readUnquotedToken();

    if (!token) {
      advance(); // Skip invalid character
      continue;
    }

    // Check for keyword operators (only for unquoted tokens)
    const upperToken = token.toUpperCase();
    if (upperToken === 'AND') {
      tokens.push({ type: 'AND' });
      continue;
    }

    if (upperToken === 'OR') {
      tokens.push({ type: 'OR' });
      continue;
    }

    if (upperToken === 'NOT') {
      tokens.push({ type: 'NOT' });
      continue;
    }

    // Try to parse as filter (unquoted token only)
    const filter = parseFilter(token);
    if (filter) {
      tokens.push(filter);
      continue;
    }

    // Otherwise, treat as text (unescape backslashes)
    tokens.push({ type: 'TEXT', value: unescapeBackslashes(token) });
  }

  tokens.push({ type: 'EOF' });

  return { tokens, issues };
}