WsParser_VS/libToken/libtoken.cpp

#include "libtoken.h"

#include <QFile>
#include <QTextStream>

using namespace lib_token;

std::shared_ptr<const IWordBase> WordReader::wordsFrom(const QString& path) const {
	auto content_list = extract_from(path);
	if (!content_list.size())
		return nullptr;

	std::shared_ptr<const IWordBase> prev_ptr = std::make_shared<const WordImpl>(content_list.last(), nullptr);
	for (auto idx = content_list.size() - 2; idx >=0; --idx) {
		auto content_ptr = content_list[idx];
		prev_ptr = std::make_shared<const WordImpl>(content_ptr, prev_ptr);
	}

	return prev_ptr;
}

QList<std::shared_ptr<const IWordBase>> WordReader::extract_from(const QString& path) const {
	QFile file(path);
	if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
		throw new TokenException(u8"Lex[0x0000]ָ<><D6B8><EFBFBD>ļ<EFBFBD><C4BC>޷<EFBFBD><DEB7>򿪣<EFBFBD>" + path);
	}
	QTextStream tin(&file);
	tin.setCodec("UTF-8");

	QList<std::shared_ptr<const IWordBase>> ret_list;
	int line_number = 1;
	while (!tin.atEnd()) {
		uint64_t relative_offset = line_number;
		relative_offset = relative_offset << 32;
		auto line = tin.readLine() + "\n";
		ret_list.append(this->parse_line(relative_offset, line_number++, line, path));
	}

	return ret_list;
}

#include <QRegExp>
QList<std::shared_ptr<const IWordBase>> WordReader::parse_line(uint64_t start_pos, int row, const QString& line_text, const QString& path) const {
	QRegExp split_char(u8"\\s");
	auto words = line_text.split(split_char, QString::SplitBehavior::SkipEmptyParts);

	QList<std::shared_ptr<const IWordBase>> primary_words;
	int columns_offset = 0;
	for (auto& w : words) {
		auto column_start = line_text.indexOf(w, columns_offset);
		auto token = std::make_shared<WordContent>(row, column_start + 1, start_pos+column_start, w, path);
		primary_words << token;

		columns_offset = column_start + w.length();
	}

	return primary_words;
}

TokenException::TokenException(const QString& message) : msg_store(message) {}

QString TokenException::message() const { return msg_store; }

WordContent::WordContent(int r, int c, uint64_t pos, const QString& t, const QString& p) 
: row_n(r), col_n(c), doc_offset(pos), text_n(t), path_p(p) {}

QString WordContent::file() const { return path_p; }

uint64_t lib_token::WordContent::position() const {
	return doc_offset;
}

QString WordContent::content() const { return text_n; }

int WordContent::row() const { return row_n; }

int WordContent::column() const { return col_n; }

std::shared_ptr<const IWordBase> WordContent::nextWord() const {
	return nullptr;
}

WordImpl::WordImpl(std::shared_ptr<const IWordBase> content, std::shared_ptr<const IWordBase> next)
: content_ptr(content), next_ptr(next) {}

QString WordImpl::file() const
{
	return content_ptr->file();
}

uint64_t lib_token::WordImpl::position() const {
	return content_ptr->position();
}

QString WordImpl::content() const
{
	return content_ptr->content();
}

int WordImpl::row() const
{
	return content_ptr->row();
}

int WordImpl::column() const
{
	return content_ptr->column();
}

std::shared_ptr<const IWordBase> WordImpl::nextWord() const
{
	return next_ptr;
}