1 2 // Copyright Ferdinand Majerech 2011-2014. 3 // Distributed under the Boost Software License, Version 1.0. 4 // (See accompanying file LICENSE_1_0.txt or copy at 5 // http://www.boost.org/LICENSE_1_0.txt) 6 7 /// YAML tokens. 8 /// Code based on PyYAML: http://www.pyyaml.org 9 module dyaml.token; 10 11 12 import std.conv; 13 14 import dyaml.encoding; 15 import dyaml.exception; 16 import dyaml.reader; 17 import dyaml.style; 18 19 20 package: 21 22 /// Token types. 23 enum TokenID : ubyte 24 { 25 Invalid = 0, /// Invalid (uninitialized) token 26 Directive, /// DIRECTIVE 27 DocumentStart, /// DOCUMENT-START 28 DocumentEnd, /// DOCUMENT-END 29 StreamStart, /// STREAM-START 30 StreamEnd, /// STREAM-END 31 BlockSequenceStart, /// BLOCK-SEQUENCE-START 32 BlockMappingStart, /// BLOCK-MAPPING-START 33 BlockEnd, /// BLOCK-END 34 FlowSequenceStart, /// FLOW-SEQUENCE-START 35 FlowMappingStart, /// FLOW-MAPPING-START 36 FlowSequenceEnd, /// FLOW-SEQUENCE-END 37 FlowMappingEnd, /// FLOW-MAPPING-END 38 Key, /// KEY 39 Value, /// VALUE 40 BlockEntry, /// BLOCK-ENTRY 41 FlowEntry, /// FLOW-ENTRY 42 Alias, /// ALIAS 43 Anchor, /// ANCHOR 44 Tag, /// TAG 45 Scalar /// SCALAR 46 } 47 48 /// Specifies the type of a tag directive token. 49 enum DirectiveType : ubyte 50 { 51 // YAML version directive. 52 YAML, 53 // Tag directive. 54 TAG, 55 // Any other directive is "reserved" for future YAML versions. 56 Reserved 57 } 58 59 /// Token produced by scanner. 60 /// 61 /// 32 bytes on 64-bit. 62 struct Token 63 { 64 @disable int opCmp(ref Token); 65 66 // 16B 67 /// Value of the token, if any. 68 /// 69 /// Values are char[] instead of string, as Parser may still change them in a few 70 /// cases. Parser casts values to strings when producing Events. 71 char[] value; 72 // 4B 73 /// Start position of the token in file/stream. 74 Mark startMark; 75 // 4B 76 /// End position of the token in file/stream. 77 Mark endMark; 78 // 1B 79 /// Token type. 80 TokenID id; 81 // 1B 82 /// Style of scalar token, if this is a scalar token. 83 ScalarStyle style; 84 // 1B 85 /// Encoding, if this is a stream start token. 86 Encoding encoding; 87 // 1B 88 /// Type of directive for directiveToken. 89 DirectiveType directive; 90 // 4B 91 /// Used to split value into 2 substrings for tokens that need 2 values (tagToken) 92 uint valueDivider; 93 94 /// Get string representation of the token ID. 95 @property string idString() @safe pure const {return id.to!string;} 96 } 97 static assert(Token.sizeof <= 32, "Token has unexpected size"); 98 99 100 @safe pure nothrow @nogc: 101 102 /// Construct a directive token. 103 /// 104 /// Params: start = Start position of the token. 105 /// end = End position of the token. 106 /// value = Value of the token. 107 /// directive = Directive type (YAML or TAG in YAML 1.1). 108 Token directiveToken(const Mark start, const Mark end, char[] value, 109 DirectiveType directive, const uint nameEnd) 110 { 111 return Token(value, start, end, TokenID.Directive, ScalarStyle.init, Encoding.init, 112 directive, nameEnd); 113 } 114 115 /// Construct a simple (no value) token with specified type. 116 /// 117 /// Params: id = Type of the token. 118 /// start = Start position of the token. 119 /// end = End position of the token. 120 Token simpleToken(TokenID id)(const Mark start, const Mark end) 121 { 122 return Token(null, start, end, id); 123 } 124 125 /// Construct a stream start token. 126 /// 127 /// Params: start = Start position of the token. 128 /// end = End position of the token. 129 /// encoding = Encoding of the stream. 130 Token streamStartToken(const Mark start, const Mark end, const Encoding encoding) 131 { 132 return Token(null, start, end, TokenID.StreamStart, ScalarStyle.Invalid, encoding); 133 } 134 135 /// Aliases for construction of simple token types. 136 alias simpleToken!(TokenID.StreamEnd) streamEndToken; 137 alias simpleToken!(TokenID.BlockSequenceStart) blockSequenceStartToken; 138 alias simpleToken!(TokenID.BlockMappingStart) blockMappingStartToken; 139 alias simpleToken!(TokenID.BlockEnd) blockEndToken; 140 alias simpleToken!(TokenID.Key) keyToken; 141 alias simpleToken!(TokenID.Value) valueToken; 142 alias simpleToken!(TokenID.BlockEntry) blockEntryToken; 143 alias simpleToken!(TokenID.FlowEntry) flowEntryToken; 144 145 /// Construct a simple token with value with specified type. 146 /// 147 /// Params: id = Type of the token. 148 /// start = Start position of the token. 149 /// end = End position of the token. 150 /// value = Value of the token. 151 /// valueDivider = A hack for TagToken to store 2 values in value; the first 152 /// value goes up to valueDivider, the second after it. 153 Token simpleValueToken(TokenID id)(const Mark start, const Mark end, char[] value, 154 const uint valueDivider = uint.max) 155 { 156 return Token(value, start, end, id, ScalarStyle.Invalid, Encoding.init, 157 DirectiveType.init, valueDivider); 158 } 159 160 /// Alias for construction of tag token. 161 alias simpleValueToken!(TokenID.Tag) tagToken; 162 alias simpleValueToken!(TokenID.Alias) aliasToken; 163 alias simpleValueToken!(TokenID.Anchor) anchorToken; 164 165 /// Construct a scalar token. 166 /// 167 /// Params: start = Start position of the token. 168 /// end = End position of the token. 169 /// value = Value of the token. 170 /// style = Style of the token. 171 Token scalarToken(const Mark start, const Mark end, char[] value, const ScalarStyle style) 172 { 173 return Token(value, start, end, TokenID.Scalar, style); 174 }