#include "lex_foundation.h" #include "tokeniimpl.h" #include using namespace Lex; TokensReader::TokensReader(QList sequence) { analysis_sequences = sequence; } QList TokensReader::getTokensOfDocument(const QFileInfo &file) { auto batch_row = 0; QList list; QFile byte_input(file.canonicalFilePath()); if (!byte_input.open(QIODevice::Text | QIODevice::ReadOnly)) throw new LexException("指定文件无法打开:" + file.canonicalFilePath()); QTextStream source(&byte_input); source.setCodec("UTF-8"); while (!source.atEnd()) { auto line = source.readLine(); list.append(get_tokens_of_line(file, line, batch_row)); batch_row++; } return list; } QList TokensReader::getTokensOfContents(const QByteArray &buff, const QFileInfo &_file) { auto batch_row = 0; QList list; QTextStream source(buff, QIODevice::ReadOnly); source.setCodec("UTF-8"); while (!source.atEnd()) { auto line = source.readLine(); list.append(get_tokens_of_line(_file, line, batch_row)); batch_row++; } return list; } QList TokensReader::get_tokens_of_line(const QFileInfo &associate, const QString &line, int row) { auto split_seqs = line.split(" ", QString::SplitBehavior::SkipEmptyParts); auto batch_column = 0; // 转换单行的内容为源列表 QList source_sequences; for (auto &it : split_seqs) { auto inst = new TokenWord(associate.canonicalFilePath()); source_sequences.append(inst); auto start_index = line.indexOf(it, batch_column); inst->reset(it, row, start_index); batch_column = start_index + it.length(); } // 对单行的所有的内容进行解析 QList results; for (auto idx = 0; idx < source_sequences.size(); ++idx) { // 对单个词语进行解析 auto inst = source_sequences[idx]; auto retv = get_token(*inst); results.append(retv); // 如果存在未解析的剩余的内容 if (retv->remains()) source_sequences.insert(idx + 1, retv->remains()); delete inst; } return results; } Token *TokensReader::get_token(const WordBase &word) { for (auto &it : analysis_sequences) { auto lex_result = it->analysis(word); if (lex_result) return lex_result; } throw new LexException(QString("指定的词语无法解析:%1 ").arg(word.content()).arg(word.row()).arg(word.column())); } LexException::LexException(const QString &msg) { this->msg_store = msg; } QString LexException::message() { return msg_store; } const char *LexException::what() const { return msg_store.toLocal8Bit(); }