From 449f89825715185cb75febba7e41dca0e41191b7 Mon Sep 17 00:00:00 2001 From: codeboss <2422523675@qq.com> Date: Thu, 27 Jun 2024 12:55:07 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E8=BF=9Btoken=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=EF=BC=8C=E6=B7=BB=E5=8A=A0doc=E2=80=94offset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- WsNovelParser.sln | 3 + WsNovelParser/WsNovelParser.vcxproj.user | 2 +- WsNovelParser/htmlprint.cpp | 4 ++ libSyntax/ast_basic.cpp | 70 ++++++++++++++++-------- libSyntax/ast_basic.h | 5 ++ libSyntax/libsyntax.cpp | 5 ++ libSyntax/libsyntax.h | 5 ++ libToken/libtoken.cpp | 18 ++++-- libToken/libtoken.h | 13 ++++- libToken/tokens_novel.cpp | 40 +++++++++----- libToken/tokens_novel.h | 5 +- 11 files changed, 124 insertions(+), 46 deletions(-) diff --git a/WsNovelParser.sln b/WsNovelParser.sln index 09f34ac..d744a57 100644 --- a/WsNovelParser.sln +++ b/WsNovelParser.sln @@ -5,7 +5,10 @@ VisualStudioVersion = 17.8.34322.80 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "WsNovelParser", "WsNovelParser\WsNovelParser.vcxproj", "{1EF577E8-D92D-4926-9207-1567137BB672}" ProjectSection(ProjectDependencies) = postProject + {1FF80476-26C9-42FB-BFF6-D587C4941964} = {1FF80476-26C9-42FB-BFF6-D587C4941964} {C3AADEB5-3695-4DF4-B8E1-D37F928F3B2F} = {C3AADEB5-3695-4DF4-B8E1-D37F928F3B2F} + {DAB406C7-174A-47C3-893C-343079396350} = {DAB406C7-174A-47C3-893C-343079396350} + {EF557F71-99AA-4F2B-A5F5-1A4518A11C19} = {EF557F71-99AA-4F2B-A5F5-1A4518A11C19} EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libToken", "libToken\libToken.vcxproj", "{DAB406C7-174A-47C3-893C-343079396350}" diff --git a/WsNovelParser/WsNovelParser.vcxproj.user b/WsNovelParser/WsNovelParser.vcxproj.user index 24aef19..ff764d4 100644 --- a/WsNovelParser/WsNovelParser.vcxproj.user +++ b/WsNovelParser/WsNovelParser.vcxproj.user @@ -3,7 +3,7 @@ $(SolutionDir)$(Platform)\$(Configuration)\ WindowsLocalDebugger - --path "D:\手作小说\科学+修仙+创造世界" --dest E:\ + --path "D:\CustomNovels\科学+修仙+创造世界" --dest E:\ --path "D:\手作小说\科学+修仙+创造世界" diff --git a/WsNovelParser/htmlprint.cpp b/WsNovelParser/htmlprint.cpp index 5b3eff4..9904a48 100644 --- a/WsNovelParser/htmlprint.cpp +++ b/WsNovelParser/htmlprint.cpp @@ -586,6 +586,7 @@ bool printer::AstGenerate::visit(std::shared_ptr s element_stack.append(dom_story); dom_story.setAttribute(u8"name", story_node->name()); + dom_story.setAttribute(u8"address", (qulonglong)story_node.get()); dom_story.setAttribute(u8"file-path", story_node->filePath()); dom_story.setAttribute(u8"sort", story_node->sort()); @@ -603,6 +604,7 @@ bool printer::AstGenerate::visit(std::shared_ptr s element_stack.append(dom_fragment); dom_fragment.setAttribute(u8"name", fragment_node->name()); + dom_fragment.setAttribute(u8"address", (qulonglong)fragment_node.get()); dom_fragment.setAttribute(u8"file-path", fragment_node->filePath()); append_tokens(dom_fragment, fragment_node); @@ -647,6 +649,7 @@ bool printer::AstGenerate::visit(std::shared_ptr s element_stack.append(dom_volume); dom_volume.setAttribute(u8"name", volume_node->name()); + dom_volume.setAttribute(u8"address", (qulonglong)volume_node.get()); dom_volume.setAttribute(u8"file-path", volume_node->filePath()); append_tokens(dom_volume, volume_node); @@ -663,6 +666,7 @@ bool printer::AstGenerate::visit(std::shared_ptr s element_stack.append(dom_article); dom_article.setAttribute(u8"name", article_node->name()); + dom_article.setAttribute(u8"address", (qulonglong)article_node.get()); dom_article.setAttribute(u8"file-path", article_node->filePath()); append_tokens(dom_article, article_node); diff --git a/libSyntax/ast_basic.cpp b/libSyntax/ast_basic.cpp index 3e8d51e..17fee14 100644 --- a/libSyntax/ast_basic.cpp +++ b/libSyntax/ast_basic.cpp @@ -8,34 +8,34 @@ using namespace lib_syntax; ExpressionElement::ExpressionElement(std::shared_ptr bind) : _expr_rule(bind) {} std::shared_ptr ExpressionElement::definedRule() const { - return _expr_rule; + return _expr_rule; } QString ExpressionElement::filePath() const { - if(!tokens_bind.size()) - throw new SyntaxException(u8"InternalError[0x0002]һյķǷЧڵ"); - - return tokens_bind.first()->file(); + if (!tokens_bind.size()) + throw new SyntaxException(u8"InternalError[0x0002]һյķǷЧڵ"); + + return tokens_bind.first()->file(); } void ExpressionElement::tokensReset(const QList>& list) { - this->tokens_bind = list; + this->tokens_bind = list; } void ExpressionElement::addToken(std::shared_ptr token_inst) { - this->tokens_bind.append(token_inst); + this->tokens_bind.append(token_inst); } QList> ExpressionElement::children() const { - return this->children_store; + return this->children_store; } void ExpressionElement::addChild(std::shared_ptr inst) { - this->children_store.append(inst); + this->children_store.append(inst); } QList> ExpressionElement::tokens() const { - return this->tokens_bind; + return this->tokens_bind; } ExpressionContext::ExpressionContext() {} @@ -46,36 +46,60 @@ QString ExpressionContext::currentFile() const { return this->current_file_path; std::shared_ptr ExpressionContext::currentInst() const { - if(expression_stack.size()) - return expression_stack.last(); + if (expression_stack.size()) + return expression_stack.last(); - return nullptr; + return nullptr; } void ExpressionContext::pushInst(std::shared_ptr current_inst) { - if(!expression_stack.size() || expression_stack.last() != current_inst) - expression_stack.append(current_inst); + if (!expression_stack.size() || expression_stack.last() != current_inst) + expression_stack.append(current_inst); } std::shared_ptr ExpressionContext::popInst() { - auto lastx = expression_stack.takeLast(); - return lastx; + auto lastx = expression_stack.takeLast(); + return lastx; } std::shared_ptr ExpressionContext::currentExpressionRule() const { - if(rule_stack.size()) - return rule_stack.last(); - return nullptr; + if (rule_stack.size()) + return rule_stack.last(); + return nullptr; } void ExpressionContext::pushExpressionRule(std::shared_ptr inst) { - if(!rule_stack.size() || rule_stack.last() != inst) - rule_stack.append(inst); + if (!rule_stack.size() || rule_stack.last() != inst) + rule_stack.append(inst); } std::shared_ptr ExpressionContext::popExpressionRule() { - return rule_stack.takeLast(); + return rule_stack.takeLast(); } + +void ExpressionContext::appendParseErrors(int start, const QString& e) { + this->errors_storage.append(std::make_tuple(start, e)); +} + +QStringList ExpressionContext::errors() const { + QStringList values; + for (auto& tp : this->errors_storage) + values.append(std::get<1>(tp)); + + return values; +} + +void ExpressionContext::clearErrors(int start) { + for (int idx = 0; idx < this->errors_storage.size(); ++idx) { + auto &tp = errors_storage[idx]; + if(std::get<0>(tp) >= start) + errors_storage.removeAt(idx--); + } +} + +QList> ExpressionContext::currentExpressionRuleStack() const { + return rule_stack; +} \ No newline at end of file diff --git a/libSyntax/ast_basic.h b/libSyntax/ast_basic.h index d5f5110..f1fd8ac 100644 --- a/libSyntax/ast_basic.h +++ b/libSyntax/ast_basic.h @@ -89,6 +89,7 @@ namespace ast_basic { QList> rule_stack; QList> expression_stack; QString current_file_path; + QList> errors_storage; public: ExpressionContext(); @@ -104,6 +105,10 @@ namespace ast_basic { std::shared_ptr currentExpressionRule() const override; void pushExpressionRule(std::shared_ptr inst) override; std::shared_ptr popExpressionRule() override; + virtual QList> currentExpressionRuleStack() const; + void appendParseErrors(int start, const QString& error_msg) override; + QStringList errors() const override; + void clearErrors(int start) override; }; } \ No newline at end of file diff --git a/libSyntax/libsyntax.cpp b/libSyntax/libsyntax.cpp index 5649b00..5767ca8 100644 --- a/libSyntax/libsyntax.cpp +++ b/libSyntax/libsyntax.cpp @@ -1,5 +1,6 @@ #include "libsyntax.h" #include "ast_basic.h" +#include using namespace lib_syntax; using namespace std; @@ -14,6 +15,10 @@ std::tuple, std::shared_ptr> if (!head) throw new InputTerminal(rt_inst->currentFile()); + if (head->content() == u8"") { + qDebug() << u8""; + } + auto match_result = define_peer->analysis(head); if (std::get<0>(match_result)) { rt_inst->currentInst()->addToken(std::get<0>(match_result)); diff --git a/libSyntax/libsyntax.h b/libSyntax/libsyntax.h index 56da438..4fc94fa 100644 --- a/libSyntax/libsyntax.h +++ b/libSyntax/libsyntax.h @@ -38,6 +38,10 @@ namespace lib_syntax { virtual void setCurrentFile(const QString &path) = 0; virtual QString currentFile() const = 0; + virtual void appendParseErrors(int start, const QString &error_msg) = 0; + virtual QStringList errors() const = 0; + virtual void clearErrors(int start) = 0; + /** * \brief ǰʽԪ. * @@ -50,6 +54,7 @@ namespace lib_syntax { virtual std::shared_ptr currentExpressionRule() const = 0; virtual void pushExpressionRule(std::shared_ptr inst) = 0; virtual std::shared_ptr popExpressionRule() = 0; + virtual QList> currentExpressionRuleStack() const = 0; }; /** diff --git a/libToken/libtoken.cpp b/libToken/libtoken.cpp index 53ed0df..247c585 100644 --- a/libToken/libtoken.cpp +++ b/libToken/libtoken.cpp @@ -30,15 +30,16 @@ QList> WordReader::extract_from(const QString& QList> ret_list; int line_number = 1; while (!tin.atEnd()) { + auto doc_pos = tin.pos(); auto line = tin.readLine() + "\n"; - ret_list.append(this->parse_line(line_number++, line, path)); + ret_list.append(this->parse_line(doc_pos, line_number++, line, path)); } return ret_list; } #include -QList> WordReader::parse_line(int row, const QString& line_text, const QString& path) const { +QList> WordReader::parse_line(int start_pos, int row, const QString& line_text, const QString& path) const { QRegExp split_char(u8"\\s"); auto words = line_text.split(split_char, QString::SplitBehavior::SkipEmptyParts); @@ -46,7 +47,7 @@ QList> WordReader::parse_line(int row, const QS int columns_offset = 0; for (auto& w : words) { auto column_start = line_text.indexOf(w, columns_offset); - auto token = std::make_shared(row, column_start + 1, w, path); + auto token = std::make_shared(row, column_start + 1, start_pos+column_start, w, path); primary_words << token; columns_offset = column_start + w.length(); @@ -59,10 +60,15 @@ TokenException::TokenException(const QString& message) : msg_store(message) {} QString TokenException::message() const { return msg_store; } -WordContent::WordContent(int r, int c, const QString& t, const QString& p) : row_n(r), col_n(c), text_n(t), path_p(p) {} +WordContent::WordContent(int r, int c, int pos, const QString& t, const QString& p) +: row_n(r), col_n(c), doc_offset(pos), text_n(t), path_p(p) {} QString WordContent::file() const { return path_p; } +int lib_token::WordContent::position() const { + return doc_offset; +} + QString WordContent::content() const { return text_n; } int WordContent::row() const { return row_n; } @@ -81,6 +87,10 @@ QString WordImpl::file() const return content_ptr->file(); } +int lib_token::WordImpl::position() const { + return content_ptr->position(); +} + QString WordImpl::content() const { return content_ptr->content(); diff --git a/libToken/libtoken.h b/libToken/libtoken.h index 8aedb75..70c4b2d 100644 --- a/libToken/libtoken.h +++ b/libToken/libtoken.h @@ -16,6 +16,11 @@ namespace lib_token { * @return */ virtual QString file() const = 0; + /** + * @brief ȡλ + * @return ʼλ + */ + virtual int position() const = 0; /** * @brief * @return @@ -44,15 +49,16 @@ namespace lib_token { */ class WordContent : public IWordBase { private: - int row_n, col_n; + int row_n, col_n, doc_offset; QString text_n, path_p; public: - WordContent(int r, int c, const QString& t, const QString& p); + WordContent(int r, int c, int pos, const QString& t, const QString& p); // WordBase interface public: virtual QString file() const override; + virtual int position() const override; virtual QString content() const override; virtual int row() const override; virtual int column() const override; @@ -69,6 +75,7 @@ namespace lib_token { // ͨ IWordBase ̳ QString file() const override; + virtual int position() const override; QString content() const override; int row() const override; int column() const override; @@ -136,7 +143,7 @@ namespace lib_token { */ class LIBTOKEN_EXPORT WordReader { private: - QList> parse_line(int row, const QString& line_text, const QString& path) const; + QList> parse_line(int start_pos, int row, const QString& line_text, const QString& path) const; QList> extract_from(const QString& path) const; public: diff --git a/libToken/tokens_novel.cpp b/libToken/tokens_novel.cpp index 77abfd6..da3673a 100644 --- a/libToken/tokens_novel.cpp +++ b/libToken/tokens_novel.cpp @@ -4,11 +4,15 @@ using namespace example_novel; using namespace lib_token; -TokenContent::TokenContent(int r, int c, const QString& t, const QString& p, std::shared_ptr paramType) - : row_n(r), col_n(c), text_n(t), path_p(p), type_def(paramType) {} +TokenContent::TokenContent(int r, int c, int pos, const QString& t, const QString& p, std::shared_ptr paramType) + : row_n(r), col_n(c), doc_offset(pos), text_n(t), path_p(p), type_def(paramType) {} QString TokenContent::file() const { return path_p; } +int lib_token::TokenContent::position() const { + return doc_offset; +} + QString TokenContent::content() const { return text_n; } int TokenContent::row() const { return row_n; } @@ -37,11 +41,13 @@ LeftBracket::analysis(std::shared_ptr content) const { if (!text.startsWith(regex())) return std::make_tuple(nullptr, content); - auto token_inst = std::make_shared(content->row(), content->column(), content->content().mid(0, regex().length()), - content->file(), shared_from_this()); + auto token_inst = std::make_shared(content->row(), content->column(), content->position(), + content->content().mid(0, regex().length()), content->file(), shared_from_this()); + auto t_remains = content->content().mid(regex().length()); if (t_remains.length() > 0) { - auto remains = std::make_shared(content->row(), content->column() + regex().length(), t_remains, content->file()); + auto remains = std::make_shared(content->row(), content->column() + regex().length(), + content->position() + regex().length(), t_remains, content->file()); return std::make_tuple(token_inst, remains); } return std::make_tuple(token_inst, nullptr); @@ -82,7 +88,8 @@ Keywords::analysis(std::shared_ptr content) const { return std::make_tuple(nullptr, content); } - auto token_inst = std::make_shared(content->row(), content->column(), content->content(), content->file(), shared_from_this()); + auto token_inst = std::make_shared(content->row(), content->column(), content->position(), + content->content(), content->file(), shared_from_this()); return std::make_tuple(token_inst, nullptr); } @@ -111,7 +118,8 @@ Numbers::analysis(std::shared_ptr content) const { if (regx.indexIn(text) == -1) return std::make_tuple(nullptr, content); - auto tinst = std::make_shared(content->row(), content->column(), content->content(), content->file(), shared_from_this()); + auto tinst = std::make_shared(content->row(), content->column(), content->position(), + content->content(), content->file(), shared_from_this()); return std::make_tuple(tinst, nullptr); } @@ -135,9 +143,11 @@ VTextSection::analysis(std::shared_ptr content) const { auto match = regx.cap(1); auto remains = content->content().mid(match.length()); - auto tinst = std::make_shared(content->row(), content->column(), match, content->file(), shared_from_this()); + auto tinst = std::make_shared(content->row(), content->column(), content->position(), + match, content->file(), shared_from_this()); if (remains.length()) { - auto t_remains = std::make_shared(content->row(), content->column(), remains, content->file()); + auto t_remains = std::make_shared(content->row(), content->column(), + content->position() + match.length(), remains, content->file()); return std::make_tuple(tinst, t_remains); } return std::make_tuple(tinst, nullptr); @@ -172,9 +182,11 @@ std::tuple, std::shared_ptr > Nam auto match = regx.cap(1); auto remains = content->content().mid(match.length()); - auto tinst = std::make_shared(content->row(), content->column(), match, content->file(), shared_from_this()); + auto tinst = std::make_shared(content->row(), content->column(), content->position(), + match, content->file(), shared_from_this()); if (remains.length()) { - auto t_remains = std::make_shared(content->row(), content->column(), remains, content->file()); + auto t_remains = std::make_shared(content->row(), content->column(), + content->position() + match.length(), remains, content->file()); return std::make_tuple(tinst, t_remains); } return std::make_tuple(tinst, nullptr); @@ -204,9 +216,11 @@ std::tuple, std::shared_ptr> Decl auto remains = content->content().mid(regex().size()); - auto tinst = std::make_shared(content->row(), content->column(), u8"#", content->file(), shared_from_this()); + auto tinst = std::make_shared(content->row(), content->column(), content->position(), + u8"#", content->file(), shared_from_this()); if (remains.length()) { - auto t_remains = std::make_shared(content->row(), content->column()+1, remains, content->file()); + auto t_remains = std::make_shared(content->row(), content->column()+1, + content->position() + regex().length(), remains, content->file()); return std::make_tuple(tinst, t_remains); } return std::make_tuple(tinst, nullptr); diff --git a/libToken/tokens_novel.h b/libToken/tokens_novel.h index 4048490..1064cbb 100644 --- a/libToken/tokens_novel.h +++ b/libToken/tokens_novel.h @@ -13,16 +13,17 @@ namespace lib_token { */ class TokenContent : public IToken { private: - int row_n, col_n; + int row_n, col_n, doc_offset; QString text_n, path_p; std::shared_ptr type_def; public: - TokenContent(int r, int c, const QString& t, const QString& p, std::shared_ptr paramType); + TokenContent(int r, int c, int pos, const QString& t, const QString& p, std::shared_ptr paramType); // WordBase interface public: virtual QString file() const override; + virtual int position() const override; virtual QString content() const override; virtual int row() const override; virtual int column() const override;