std::string prefix = "xmlns";

if (!ns.prefix.empty()) {

prefix += ':';

prefix += ns.prefix;

}

xmlTextWriterWriteAttribute(xout, BAD_CAST prefix.data(), BAD_CAST ns.uri.data());

xmlOutputBuffer* output_buffer,

const char* language,

const char* xml_enc,

OPTION_TYPE& op,

const Attributes& attributes,

const std::optional<std::pair<std::string, std::string>>& processing_instruction,

size_t ts)

: output_buffer(output_buffer), input(ints), unit_language(language),

options(op), xml_encoding(xml_enc), attributes(attributes), processing_instruction(processing_instruction),

tabsize(ts)

{

// open the output text writer stream

xout = xmlNewTextWriter(output_buffer);

// start with the default

namespaces = default_namespaces;

// merge in the other namespaces

namespaces += otherns;

if (depth == 0)

archiveNamespaces = namespaces;

if (xout) {

if (startedOutput)

xmlTextWriterEndDocument(xout);

xmlFreeTextWriter(xout);

xout = 0;

output_buffer = 0;

} else if (output_buffer) {

xmlOutputBufferClose(output_buffer);

output_buffer = 0;

}

const char* version, const char* timestamp, const char* hash, const char* encoding) {

// store attributes so that first occurrence of unit element will be correct

unit_revision = revision;

unit_url = url;

unit_filename = filename;

unit_version = version;

unit_timestamp = timestamp;

unit_language = language;

unit_hash = hash;

unit_encoding = encoding;

Position currentPosition;

while (1) {

const antlr::RefToken& token = input->nextToken();

if (!isoption(options, SRCML_PARSER_OPTION_POSITION)) {

if (token->getType() == antlr::Token::EOF_TYPE)

break;

outputToken(token);

} else {

if (token->getType() == antlr::Token::EOF_TYPE) {

while (!tokenQueue.empty()) {

outputToken(tokenQueue.front());

tokenQueue.pop_front();

}

break;

}

// text token

auto search = process.find(token->getType());

if (!(search != process.end() && search->second.name)) {

// update the text position

currentPosition.append(token->getText(), tabsize);

// save the text

tokenQueue.push_back(token);

// start token but not empty

} else if (isstart(token) && !isempty(token)) {

// set the start line/column

token->setLine(currentPosition.line);

token->setColumn(currentPosition.column + 1);

// save open start elements

startElementStack.push(token);

// save the start element

tokenQueue.push_back(token);

// start token but empty

} else if (isstart(token) && isempty(token)) {

// save the empty element

tokenQueue.push_back(token);

// end token

} else if (isend(token)) {

// most recent start token that will match the current end token

auto matchingStartElement = startElementStack.top();

startElementStack.pop();

// set the end line/column

srcMLToken* qetoken = static_cast<srcMLToken*>(&(*matchingStartElement));

qetoken->endline = currentPosition.line;

qetoken->endcolumn = currentPosition.column;

// save the end token

tokenQueue.push_back(token);

// if there are no open elements, then we are at the root

// and can output all elements in the queue

if (startElementStack.empty()) {

while (!tokenQueue.empty()) {

outputToken(tokenQueue.front());

tokenQueue.pop_front();

}

}

}

}

}

// issue the xml declaration, but only if we want to

if (depth == 0 && !isoption(options, SRCML_PARSER_OPTION_NO_XML_DECL))

xmlTextWriterStartDocument(xout, XML_VERSION.data(), xml_encoding, XML_DECLARATION_STANDALONE.data());

if (depth == 0 && processing_instruction) {

startedOutput = true;

xmlTextWriterStartPI(xout, BAD_CAST processing_instruction->first.data());

xmlTextWriterWriteString(xout, BAD_CAST processing_instruction->second.data());

xmlTextWriterEndPI(xout);

xmlTextWriterWriteString(xout, BAD_CAST "\n");

}

// based on options, turn on specific namespaces (i.e., mark as used)

if (isoption(options, SRCML_OPTION_CPP_DECLARED))

findNSURI(namespaces, SRCML_CPP_NS_URI)->flags |= NS_USED;

if (isoption(options, SRCML_PARSER_OPTION_POSITION))

findNSURI(namespaces, SRCML_POSITION_NS_URI)->flags |= NS_USED;

if (isoption(options, SRCML_OPTION_DEBUG))

findNSURI(namespaces, SRCML_ERROR_NS_URI)->flags |= NS_USED;

for (const auto& ns : namespaces) {

// output standard namespaces for outer unit or non-archive unit

if (depth == 0 || !isoption(options, SRCML_OPTION_ARCHIVE)) {

// must be required, or on the root and used

if ((ns.flags & NS_STANDARD) && ((ns.flags & NS_REQUIRED) || ((ns.flags & NS_ROOT) && (ns.flags & NS_USED)))) {

srcMLTextWriterWriteNamespace(xout, ns);

continue;

}

// must be user registered

if (ns.flags & NS_REGISTERED && !(ns.flags & NS_STANDARD)) {

srcMLTextWriterWriteNamespace(xout, ns);

continue;

}

}

// output standard namespaces for inner unit or non-archive unit

if (depth == 1 || !isoption(options, SRCML_OPTION_ARCHIVE)) {

// must be required, must not be on the root, and must be used

if ((ns.flags & NS_STANDARD) && !(ns.flags & NS_ROOT) && !(ns.flags & NS_REQUIRED) && (ns.flags & NS_USED)) {

srcMLTextWriterWriteNamespace(xout, ns);

continue;

} else if ((ns.flags & NS_REGISTERED) && (findNSURI(archiveNamespaces, ns.uri) == archiveNamespaces.end())) {

srcMLTextWriterWriteNamespace(xout, ns);

continue;

}

}

}

const char* url, const char* filename,

const char* version, const char* timestamp,

const char* hash,

const char* encoding,

const Attributes& unit_attributes,

bool output_macrolist) {

// go with default encoding

if (!encoding || encoding[0] == '\0') {

encoding = "UTF-8";

}

startedOutput = true;

// start of main tag

std::string_view unitprefix = namespaces[SRC].prefix;

xmlTextWriterStartElementNS(xout, BAD_CAST (!unitprefix.empty() ? unitprefix.data() : 0), BAD_CAST "unit", 0);

// output namespaces for root and nested units

if (isoption(options, SRCML_OPTION_NAMESPACE_DECL)) {

outputNamespaces();

}

// options

std::string soptions;

if (isoption(options, SRCML_PARSER_OPTION_CPP_TEXT_ELSE))

soptions = "CPP_TEXT_ELSE";

if (isoption(options, SRCML_PARSER_OPTION_CPP_MARKUP_IF0)) {

if (!soptions.empty())

soptions += ',';

soptions = "CPP_MARKUP_IF0";

}

if (isoption(options, SRCML_OPTION_LINE)) {

if (!soptions.empty())

soptions += ',';

soptions = "LINE";

}

// revision standard attribute

if (revision) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_REVISION.data(), BAD_CAST revision);

}

// language standard attribute

if (language) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_LANGUAGE.data(), BAD_CAST language);

}

// url standard attribute

if (url) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_URL.data(), BAD_CAST url);

}

// filename standard attribute

if (filename) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_FILENAME.data(), BAD_CAST filename);

}

// version standard attribute

if (version) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_VERSION.data(), BAD_CAST version);

}

// tabsize standard attribute

if (isoption(options, SRCML_PARSER_OPTION_POSITION)) {

std::string tabattribute(namespaces[POS].getPrefix());

if (!tabattribute.empty())

tabattribute += ':';

for (auto c : "tabs"sv)

tabattribute += c;

auto stabsize = std::to_string(tabsize);

xmlTextWriterWriteAttribute(xout, BAD_CAST tabattribute.data(), BAD_CAST stabsize.data());

}

// timestamp standard attribute

if (timestamp) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_TIMESTAMP.data(), BAD_CAST timestamp);

}

// hash standard attribute

if (hash) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_HASH.data(), BAD_CAST hash);

}

// encoding standard attribute

if (isoption(options, SRCML_PARSER_OPTION_STORE_ENCODING)) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_SOURCE_ENCODING.data(), BAD_CAST encoding);

}

// options standard attribute

if (depth == 0 && !soptions.empty()) {

xmlTextWriterWriteAttribute(xout, BAD_CAST UNIT_ATTRIBUTE_OPTIONS.data(), BAD_CAST soptions.data());

}

// custom attributes

for (const auto& attribute : unit_attributes) {

std::string qName;

if (!attribute.prefix.empty()) {

qName += attribute.prefix;

qName += ':';

}

qName += attribute.name;

xmlTextWriterWriteAttribute(xout, BAD_CAST qName.data(), BAD_CAST attribute.value.data());

}

if (output_macrolist)

outputMacroList();

if (depth == 0)

depth = 1;

for(std::vector<std::string>::size_type i = 0; i < user_macro_list.size(); i += 2) {

xmlTextWriterStartElement(xout, BAD_CAST "macro-list");

xmlTextWriterWriteAttribute(xout, BAD_CAST "token", BAD_CAST user_macro_list[i].data());

xmlTextWriterWriteAttribute(xout, BAD_CAST "type", BAD_CAST user_macro_list[i + 1].data());

xmlTextWriterEndElement(xout);

}

// skip processing empty stings

if (str.empty())

return;

// output any '<', '>', or '&', or any text before

std::size_t p = 0;

auto lastp = p;

while ((p = str.find_first_of("<>&"sv, p)) != str.npos) {

// output section before

xmlTextWriterWriteRawLen(xout, BAD_CAST (unsigned char*) &str.data()[lastp], p - lastp);

// output special characters

if (str[p] == '<') {

xmlTextWriterWriteRawLen(xout, BAD_CAST "&lt;", "&lt;"sv.size());

} else if (str[p] == '>') {

xmlTextWriterWriteRawLen(xout, BAD_CAST "&gt;", "&gt;"sv.size());

} else if (str[p] == '&') {

xmlTextWriterWriteRawLen(xout, BAD_CAST "&amp;", "&amp;"sv.size());

}

++p;

lastp = p;

}

// output remaining text after last '<', '>', or '&', or the entire string if these do not occur

xmlTextWriterWriteRawLen(xout, BAD_CAST (unsigned char*) &str.data()[lastp], str.size() - lastp);

srcMLToken* stoken = static_cast<srcMLToken*>(&(*token));

// how we detect empty elements: the position is wrong

if (stoken->endline < stoken->getLine() || (stoken->endline == stoken->getLine() && stoken->endcolumn < stoken->getColumn()))

return;

thread_local const std::string prefix(namespaces[POS].prefix);

thread_local const std::string startAttribute = " " + prefix + (!prefix.empty() ? ":" : "") + "start=\"";

thread_local const std::string endAttribute = " " + prefix + (!prefix.empty() ? ":" : "") + "end=\"";

// highly optimized as this is output for every start tag

// position start attribute, e.g. pos:start="1:4"

xmlOutputBufferWrite(output_buffer, (int) startAttribute.size(), startAttribute.data());

xmlOutputBufferWriteString(output_buffer, positoa(token->getLine()));

xmlOutputBufferWrite(output_buffer, 1, ":");

xmlOutputBufferWriteString(output_buffer, positoa(token->getColumn()));

xmlOutputBufferWrite(output_buffer, 1, "\"");

xmlOutputBufferFlush(output_buffer);

// position end attribute, e.g. pos:end="2:1"

xmlOutputBufferWrite(output_buffer, (int) endAttribute.size(), endAttribute.data());

if (token->getLine() > stoken->endline) {

xmlOutputBufferWriteString(output_buffer, "INVALID_POS(");

}

xmlOutputBufferWriteString(output_buffer, positoa(stoken->endline));

if (token->getLine() > stoken->endline) {

xmlOutputBufferWriteString(output_buffer, ")");

}

xmlOutputBufferWrite(output_buffer, 1, ":");

xmlOutputBufferWriteString(output_buffer, positoa(stoken->endcolumn));

xmlOutputBufferWrite(output_buffer, 1, "\"");

const char* attr_name2, const char* attr_value2) {

// no name, no token

if (name[0] == 0)

return;

bool isposition = isoption(options, SRCML_PARSER_OPTION_POSITION);

if (isstart(token) || isempty(token)) {

// empty prefixes have to be null for output

if (prefix[0] == '\0')

prefix = 0;

xmlTextWriterStartElementNS(xout, BAD_CAST prefix, BAD_CAST name, 0);

if (attr_name1)

xmlTextWriterWriteAttribute(xout, BAD_CAST attr_name1, BAD_CAST attr_value1);

if (attr_name2)

xmlTextWriterWriteAttribute(xout, BAD_CAST attr_name2, BAD_CAST attr_value2);

if (isposition) {

// save start and end positions for any future previous type

if (token->getType() == srcMLParserTokenTypes::STYPE) {

lastTypeStartPosition = Position(token->getLine(), token->getColumn());

srcMLToken* qetoken = static_cast<srcMLToken*>(&(*token));

lastTypeEndPosition = Position(qetoken->endline, qetoken->endcolumn);

}

// previous type get start and end positions from previous, well type

if (token->getType() == srcMLParserTokenTypes::STYPEPREV) {

token->setLine(lastTypeStartPosition.line);

token->setColumn(lastTypeStartPosition.column);

srcMLToken* qetoken = static_cast<srcMLToken*>(&(*token));

qetoken->endline = lastTypeEndPosition.line;

qetoken->endcolumn = lastTypeEndPosition.column;

}

// if position attributes for non-empty start elements

if (!isempty(token) || token->getType() == srcMLParserTokenTypes::STYPEPREV)

addPosition(token);

}

}

if (!isstart(token) || isempty(token)) {

xmlTextWriterEndElement(xout);

}

// unit element is handled specially

if (srcMLParserTokenTypes::SUNIT == token->getType())

return;

// find the token in the element map. If found and it has a name, then process the token

auto search = process.find(token->getType());

if (search != process.end() && search->second.name) {

const Element& eparts = search->second;

// process the token using the fields in the element

processToken(token, eparts.name,

// use getPrefix() to record that this prefix was used

namespaces[eparts.prefix].getPrefix().data(),

eparts.attr_name,

// if attribute name and no value, then take text from token

eparts.attr_name && eparts.attr_value ? eparts.attr_value : token->getText().data(),

eparts.attr2_name,

eparts.attr2_value);

return;

}

// remainder are treated as text tokens

processText(token);