#include "lex_foundation.h" #include "token_impls.h" #include #include using namespace Lex; TokenReader::TokenReader(QList sequence, const QFileInfo &target) : line_number(0), target_info(target), bin_source(nullptr), content_stack(""), line_source(nullptr), analysis_sequences(sequence) { this->bin_source = new QFile(target.absoluteFilePath()); if (!this->bin_source->open(QIODevice::ReadOnly | QIODevice::Text)) throw new WsBaseException("指定源文件无法打开!"); this->line_source = new QTextStream(this->bin_source); } TokenReader::TokenReader(QList sequence, const QFileInfo &target, const QString &content) : line_number(0), target_info(target), bin_source(nullptr), content_stack(content), line_source(nullptr), analysis_sequences(sequence) { this->line_source = new QTextStream(&content_stack, QIODevice::ReadOnly); } TokenReader::~TokenReader() { for (auto &ins : tokens_buffer) delete ins; for (auto &ins : analysis_sequences) delete ins; delete line_source; delete bin_source; } Token *TokenReader::read() { if (this->tokens_buffer.count() < 1) this->tokens_buffer.append(get_tokens_of_line(this->target_info, *this->line_source)); if (this->tokens_buffer.count() < 1) return nullptr; return this->tokens_buffer.takeAt(0); } void TokenReader::tokenRemove(uint index) { if (this->tokens_buffer.count() <= index) throw new WsBaseException("索引超出当前序列最大容量"); delete this->tokens_buffer.takeAt(index); } Token *TokenReader::tokenPeak(uint index) { if (this->tokens_buffer.count() <= index) this->tokens_buffer.append(get_tokens_of_line(this->target_info, *this->line_source)); if (this->tokens_buffer.count() <= index) throw new WsBaseException("目标获取数量超出最大能力"); return this->tokens_buffer.at(index); } QList TokenReader::get_tokens_of_line(const QFileInfo &associate, QTextStream &lines_source) { const QString line = line_source->readLine(); auto split_seqs = line.split(QRegExp("\\s\\t"), QString::SplitBehavior::SkipEmptyParts); split_seqs.append("\n"); auto batch_column = 0; // 转换单行的内容为词组列表 QList word_source; for (auto &it : split_seqs) { auto inst = new TokenWord(associate.canonicalFilePath()); word_source.append(inst); auto start_index = line.indexOf(it, batch_column); inst->reset(it, line_number, start_index); batch_column = start_index + it.length(); } // 对单行的所有的内容进行解析 QList token_results; for (auto idx = 0; idx < word_source.size(); ++idx) { // 对单个词语进行解析 auto inst = word_source[idx]; auto retv = get_token(*inst); token_results.append(retv); // 如果存在未解析的剩余的内容 if (retv->remains()) word_source.insert(idx + 1, retv->remains()); } // 删除所有的词元列表 for (auto &token_ins : word_source) delete token_ins; line_number++; return token_results; } Token *TokenReader::get_token(const WordBase &word) { for (auto &it : analysis_sequences) { auto lex_result = it->analysis(word); if (lex_result) return lex_result; } throw new WsBaseException(QString("指定的词语无法解析:%1 ").arg(word.content()).arg(word.row()).arg(word.column())); } WsBaseException::WsBaseException(const QString &msg) { this->msg_store = msg; } QString WsBaseException::message() const { return msg_store; } const char *WsBaseException::what() const { return msg_store.toLocal8Bit(); }