token流精简

libToken/libtoken.cpp

@@ -7,7 +7,21 @@ using namespace lib_token;
 
 TokenReader::TokenReader(const QList<std::shared_ptr<const TokenDefine>> rulers) : rules_store(rulers) {}
 
-QList<std::shared_ptr<const lib_token::Token>> lib_token::TokenReader::extractFrom(const QString& path) const {
+std::shared_ptr<const Token> lib_token::TokenReader::tokensWithin(const QString& path) const {
+	auto content_list = extract_from(path);
+	if (!content_list.size())
+		return nullptr;
+
+	std::shared_ptr<const Token> prev_ptr = std::make_shared<const TokenImpl>(content_list.last(), nullptr);
+	for (auto idx = content_list.size() - 2; idx >=0; --idx) {
+		auto content_ptr = content_list[idx];
+		prev_ptr = std::make_shared<const TokenImpl>(content_ptr, prev_ptr);
+	}
+
+	return prev_ptr;
+}
+
+QList<std::shared_ptr<const Token>> TokenReader::extract_from(const QString& path) const {
 	QFile file(path);
 	if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
 		throw new TokenException(u8"Lex[0x0000]指定文件无法打开：" + path);
@@ -77,15 +91,53 @@ int WordPeaks::row() const { return row_n; }
 
 int WordPeaks::column() const { return col_n; }
 
-TokenInst::TokenInst(int r, int c, const QString& t, const QString& p, std::shared_ptr<const TokenDefine> type)
+TokenContent::TokenContent(int r, int c, const QString& t, const QString& p, std::shared_ptr<const TokenDefine> type)
 	: row_n(r), col_n(c), text_n(t), path_p(p), type_def(type) {}
 
-QString TokenInst::file() const { return path_p; }
+QString TokenContent::file() const { return path_p; }
 
-QString TokenInst::content() const { return text_n; }
+QString TokenContent::content() const { return text_n; }
 
-int TokenInst::row() const { return row_n; }
+int TokenContent::row() const { return row_n; }
 
-int TokenInst::column() const { return col_n; }
+int TokenContent::column() const { return col_n; }
 
-std::shared_ptr<const TokenDefine> TokenInst::define() const { return this->type_def; }
+std::shared_ptr<const TokenDefine> TokenContent::define() const { return this->type_def; }
+
+std::shared_ptr<const Token> TokenContent::nextToken() const
+{
+	return nullptr;
+}
+
+TokenImpl::TokenImpl(std::shared_ptr<const Token> content, std::shared_ptr<const Token> next)
+	: content_ptr(content), next_element(next) {}
+
+QString TokenImpl::file() const
+{
+	return content_ptr->file();
+}
+
+QString TokenImpl::content() const
+{
+	return content_ptr->content();
+}
+
+int TokenImpl::row() const
+{
+	return content_ptr->row();
+}
+
+int TokenImpl::column() const
+{
+	return content_ptr->column();
+}
+
+std::shared_ptr<const TokenDefine> TokenImpl::define() const
+{
+	return content_ptr->define();
+}
+
+std::shared_ptr<const Token> TokenImpl::nextToken() const
+{
+	return next_element;
+}

libToken/libtoken.h

@@ -7,6 +7,17 @@
 
 namespace lib_token {
 	class TokenDefine;
+	/**
+	 * @brief ½âÎöÒì³£
+	 */
+	class LIBTOKEN_EXPORT TokenException {
+	private:
+		QString msg_store;
+
+	public:
+		TokenException(const QString& message);
+		virtual QString message() const;
+	};
 
 	/**
 	 * @brief 源码词语实例
@@ -45,6 +56,7 @@ namespace lib_token {
 		 * @return
 		 */
 		virtual std::shared_ptr<const TokenDefine> define() const = 0;
+		virtual std::shared_ptr<const Token> nextToken() const = 0;
 	};
 
 	/**
@@ -71,22 +83,11 @@ namespace lib_token {
 		virtual std::tuple<std::shared_ptr<const Token>, std::shared_ptr<const WordBase>> analysis(std::shared_ptr<const WordBase> content) const = 0;
 	};
 
-    /**
-     * @brief ½âÎöÒì³£
-     */
-    class LIBTOKEN_EXPORT TokenException {
-    private:
-        QString msg_store;
-
-    public:
-        TokenException(const QString& message);
-        virtual QString message() const;
-    };
 
 	/**
 	 * @brief 未解析原始词语
 	 */
-    class LIBTOKEN_EXPORT WordPeaks : public WordBase {
+	class WordPeaks : public WordBase {
 	private:
 		int row_n, col_n;
 		QString text_n, path_p;
@@ -105,14 +106,14 @@ namespace lib_token {
 	/**
 	 * @brief 词法解析成果
 	 */
-    class LIBTOKEN_EXPORT TokenInst : public Token {
+	class TokenContent : public Token {
 	private:
 		int row_n, col_n;
 		QString text_n, path_p;
 		std::shared_ptr<const TokenDefine> type_def;
 
 	public:
-        TokenInst(int r, int c, const QString& t, const QString& p, std::shared_ptr<const TokenDefine> type);
+		TokenContent(int r, int c, const QString& t, const QString& p, std::shared_ptr<const TokenDefine> type);
 
 		// WordBase interface
 	public:
@@ -124,6 +125,24 @@ namespace lib_token {
 		// Token interface
 	public:
 		virtual std::shared_ptr<const TokenDefine> define() const override;
+		virtual std::shared_ptr<const Token> nextToken() const override;
+	};
+
+	class TokenImpl : public Token {
+	private:
+		std::shared_ptr<const Token> content_ptr;
+		std::shared_ptr<const Token> next_element;
+
+	public:
+		TokenImpl(std::shared_ptr<const Token> content, std::shared_ptr<const Token> next);
+
+		// ͨ¹ý Token ¼Ì³Ð
+		QString file() const override;
+		QString content() const override;
+		int row() const override;
+		int column() const override;
+		std::shared_ptr<const TokenDefine> define() const override;
+		std::shared_ptr<const Token> nextToken() const override;
 	};
 
 	/**
@@ -135,11 +154,11 @@ namespace lib_token {
 
 		QList<std::shared_ptr<const Token>> parse_line(int row, const QString& line_text, const QString& path) const;
 		std::tuple<std::shared_ptr<const Token>, std::shared_ptr<const WordBase>> parse_token(std::shared_ptr<const WordBase> word) const;
+		QList<std::shared_ptr<const Token>> extract_from(const QString& path) const;
 
 	public:
 		TokenReader(const QList<std::shared_ptr<const TokenDefine>> rulers);
-
+		std::shared_ptr<const Token> tokensWithin(const QString &path) const;
-        QList<std::shared_ptr<const Token>> extractFrom(const QString& path) const;
 	};
 
 } // namespace lib_token

libToken/tokens_novel.cpp

@@ -1,22 +1,23 @@
 #include "tokens_novel.h"
 
 using namespace example_novel;
+using namespace lib_token;
 
 QString LeftBracket::name() const { return u8"left-bracket"; }
 
 QString LeftBracket::regex() const { return u8"{"; }
 
-std::tuple<std::shared_ptr<const lib_token::Token>, std::shared_ptr<const lib_token::WordBase>>
+std::tuple<std::shared_ptr<const Token>, std::shared_ptr<const WordBase>>
-LeftBracket::analysis(std::shared_ptr<const lib_token::WordBase> content) const {
+LeftBracket::analysis(std::shared_ptr<const WordBase> content) const {
     auto text = content->content();
     if (!text.startsWith(regex()))
         return std::make_tuple(nullptr, content);
 
-    auto token_inst = std::make_shared<lib_token::TokenInst>(content->row(), content->column(), content->content().mid(0, regex().length()),
+    auto token_inst = std::make_shared<TokenContent>(content->row(), content->column(), content->content().mid(0, regex().length()),
         content->file(), shared_from_this());
     auto t_remains = content->content().mid(regex().length());
     if (t_remains.length() > 0) {
-        auto remains = std::make_shared<lib_token::WordPeaks>(content->row(), content->column() + regex().length(), t_remains, content->file());
+        auto remains = std::make_shared<WordPeaks>(content->row(), content->column() + regex().length(), t_remains, content->file());
         return std::make_tuple(token_inst, remains);
     }
     return std::make_tuple(token_inst, nullptr);
@@ -36,13 +37,13 @@ QString Keywords::name() const { return name_store; }
 
 QString Keywords::regex() const { return value_store; }
 
-std::tuple<std::shared_ptr<const lib_token::Token>, std::shared_ptr<const lib_token::WordBase>>
+std::tuple<std::shared_ptr<const Token>, std::shared_ptr<const WordBase>>
-Keywords::analysis(std::shared_ptr<const lib_token::WordBase> content) const {
+Keywords::analysis(std::shared_ptr<const WordBase> content) const {
     if (content->content() != regex()) {
         return std::make_tuple(nullptr, content);
     }
 
-    auto token_inst = std::make_shared<lib_token::TokenInst>(content->row(), content->column(), content->content(), content->file(), shared_from_this());
+    auto token_inst = std::make_shared<TokenContent>(content->row(), content->column(), content->content(), content->file(), shared_from_this());
     return std::make_tuple(token_inst, nullptr);
 }
 
@@ -54,14 +55,14 @@ QString Numbers::name() const { return u8"numbers"; }
 
 QString Numbers::regex() const { return u8"^([0-9]+)$"; }
 
-std::tuple<std::shared_ptr<const lib_token::Token>, std::shared_ptr<const lib_token::WordBase>>
+std::tuple<std::shared_ptr<const Token>, std::shared_ptr<const WordBase>>
-Numbers::analysis(std::shared_ptr<const lib_token::WordBase> content) const {
+Numbers::analysis(std::shared_ptr<const WordBase> content) const {
     auto text = content->content();
     QRegExp regx(regex());
     if (regx.indexIn(text) == -1)
         return std::make_tuple(nullptr, content);
 
-    auto tinst = std::make_shared<lib_token::TokenInst>(content->row(), content->column(), content->content(), content->file(), shared_from_this());
+    auto tinst = std::make_shared<TokenContent>(content->row(), content->column(), content->content(), content->file(), shared_from_this());
     return std::make_tuple(tinst, nullptr);
 }
 
@@ -69,8 +70,8 @@ QString VTextSection::name() const { return u8"text-section"; }
 
 QString VTextSection::regex() const { return u8"^([^\\{\\}\\n@&]+)"; }
 
-std::tuple<std::shared_ptr<const lib_token::Token>, std::shared_ptr<const lib_token::WordBase>>
+std::tuple<std::shared_ptr<const Token>, std::shared_ptr<const WordBase>>
-VTextSection::analysis(std::shared_ptr<const lib_token::WordBase> content) const {
+VTextSection::analysis(std::shared_ptr<const WordBase> content) const {
     auto text = content->content();
     QRegExp regx(regex());
     if (regx.indexIn(text) == -1) {
@@ -80,9 +81,9 @@ VTextSection::analysis(std::shared_ptr<const lib_token::WordBase> content) const
     auto match = regx.cap(1);
     auto remains = content->content().mid(match.length());
 
-    auto tinst = std::make_shared<lib_token::TokenInst>(content->row(), content->column(), match, content->file(), shared_from_this());
+    auto tinst = std::make_shared<TokenContent>(content->row(), content->column(), match, content->file(), shared_from_this());
     if (remains.length()) {
-        auto t_remains = std::make_shared<lib_token::WordPeaks>(content->row(), content->column(), remains, content->file());
+        auto t_remains = std::make_shared<WordPeaks>(content->row(), content->column(), remains, content->file());
         return std::make_tuple(tinst, t_remains);
     }
     return std::make_tuple(tinst, nullptr);
@@ -96,7 +97,7 @@ QString NameSection::name() const { return u8"name-section"; }
 
 QString NameSection::regex() const { return u8"^([^:\\{\\}\\n@&][^\\{\\}\\n@&]*)"; }
 
-std::tuple<std::shared_ptr<const lib_token::Token>, std::shared_ptr<const lib_token::WordBase> > NameSection::analysis(std::shared_ptr<const lib_token::WordBase> content) const
+std::tuple<std::shared_ptr<const Token>, std::shared_ptr<const WordBase> > NameSection::analysis(std::shared_ptr<const WordBase> content) const
 {
     auto text = content->content();
     QRegExp regx(regex());
@@ -107,9 +108,9 @@ std::tuple<std::shared_ptr<const lib_token::Token>, std::shared_ptr<const lib_to
     auto match = regx.cap(1);
     auto remains = content->content().mid(match.length());
 
-    auto tinst = std::make_shared<lib_token::TokenInst>(content->row(), content->column(), match, content->file(), shared_from_this());
+    auto tinst = std::make_shared<TokenContent>(content->row(), content->column(), match, content->file(), shared_from_this());
     if (remains.length()) {
-        auto t_remains = std::make_shared<lib_token::WordPeaks>(content->row(), content->column(), remains, content->file());
+        auto t_remains = std::make_shared<WordPeaks>(content->row(), content->column(), remains, content->file());
         return std::make_tuple(tinst, t_remains);
     }
     return std::make_tuple(tinst, nullptr);