View Javadoc

    1   /**
    2    * Copyright (c) 2008-2011, http://www.snakeyaml.org
    3    *
    4    * Licensed under the Apache License, Version 2.0 (the "License");
    5    * you may not use this file except in compliance with the License.
    6    * You may obtain a copy of the License at
    7    *
    8    *     http://www.apache.org/licenses/LICENSE-2.0
    9    *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16
    17  package org.pyyaml;
    18
    19  import java.io.File;
    20  import java.io.FileInputStream;
    21  import java.io.FileNotFoundException;
    22  import java.io.IOException;
    23  import java.io.InputStream;
    24  import java.util.ArrayList;
    25  import java.util.HashMap;
    26  import java.util.List;
    27  import java.util.Map;
    28
    29  import org.yaml.snakeyaml.reader.StreamReader;
    30  import org.yaml.snakeyaml.reader.UnicodeReader;
    31  import org.yaml.snakeyaml.scanner.Scanner;
    32  import org.yaml.snakeyaml.scanner.ScannerImpl;
    33  import org.yaml.snakeyaml.tokens.AliasToken;
    34  import org.yaml.snakeyaml.tokens.AnchorToken;
    35  import org.yaml.snakeyaml.tokens.BlockEndToken;
    36  import org.yaml.snakeyaml.tokens.BlockEntryToken;
    37  import org.yaml.snakeyaml.tokens.BlockMappingStartToken;
    38  import org.yaml.snakeyaml.tokens.BlockSequenceStartToken;
    39  import org.yaml.snakeyaml.tokens.DirectiveToken;
    40  import org.yaml.snakeyaml.tokens.DocumentEndToken;
    41  import org.yaml.snakeyaml.tokens.DocumentStartToken;
    42  import org.yaml.snakeyaml.tokens.FlowEntryToken;
    43  import org.yaml.snakeyaml.tokens.FlowMappingEndToken;
    44  import org.yaml.snakeyaml.tokens.FlowMappingStartToken;
    45  import org.yaml.snakeyaml.tokens.FlowSequenceEndToken;
    46  import org.yaml.snakeyaml.tokens.FlowSequenceStartToken;
    47  import org.yaml.snakeyaml.tokens.KeyToken;
    48  import org.yaml.snakeyaml.tokens.ScalarToken;
    49  import org.yaml.snakeyaml.tokens.StreamEndToken;
    50  import org.yaml.snakeyaml.tokens.StreamStartToken;
    51  import org.yaml.snakeyaml.tokens.TagToken;
    52  import org.yaml.snakeyaml.tokens.Token;
    53  import org.yaml.snakeyaml.tokens.ValueToken;
    54
    55  /**
    56   * @see imported from PyYAML
    57   */
    58  public class PyTokensTest extends PyImportTest {
    59
    60      public void testTokens() throws FileNotFoundException {
    61          Map<Class<?>, String> replaces = new HashMap<Class<?>, String>();
    62          replaces.put(DirectiveToken.class, "%");
    63          replaces.put(DocumentStartToken.class, "---");
    64          replaces.put(DocumentEndToken.class, "...");
    65          replaces.put(AliasToken.class, "*");
    66          replaces.put(AnchorToken.class, "&");
    67          replaces.put(TagToken.class, "!");
    68          replaces.put(ScalarToken.class, "_");
    69          replaces.put(BlockSequenceStartToken.class, "[[");
    70          replaces.put(BlockMappingStartToken.class, "{{");
    71          replaces.put(BlockEndToken.class, "]}");
    72          replaces.put(FlowSequenceStartToken.class, "[");
    73          replaces.put(FlowSequenceEndToken.class, "]");
    74          replaces.put(FlowMappingStartToken.class, "{");
    75          replaces.put(FlowMappingEndToken.class, "}");
    76          replaces.put(BlockEntryToken.class, ",");
    77          replaces.put(FlowEntryToken.class, ",");
    78          replaces.put(KeyToken.class, "?");
    79          replaces.put(ValueToken.class, ":");
    80          //
    81          File[] tokensFiles = getStreamsByExtension(".tokens");
    82          assertTrue("No test files found.", tokensFiles.length > 0);
    83          for (int i = 0; i < tokensFiles.length; i++) {
    84              String name = tokensFiles[i].getName();
    85              int position = name.lastIndexOf('.');
    86              String dataName = name.substring(0, position) + ".data";
    87              //
    88              String tokenFileData = getResource(name);
    89              String[] split = tokenFileData.split("\\s+");
    90              List<String> tokens2 = new ArrayList<String>();
    91              for (int j = 0; j < split.length; j++) {
    92                  tokens2.add(split[j]);
    93              }
    94              //
    95              List<String> tokens1 = new ArrayList<String>();
    96              StreamReader reader = new StreamReader(new UnicodeReader(new FileInputStream(
    97                      getFileByName(dataName))));
    98              Scanner scanner = new ScannerImpl(reader);
    99              try {
    100                 while (scanner.checkToken(new Token.ID[0])) {
    101                     Token token = scanner.getToken();
    102                     if (!(token instanceof StreamStartToken || token instanceof StreamEndToken)) {
    103                         String replacement = replaces.get(token.getClass());
    104                         tokens1.add(replacement);
    105                     }
    106                 }
    107                 assertEquals(tokenFileData, tokens1.size(), tokens2.size());
    108                 assertEquals(tokens1, tokens2);
    109             } catch (RuntimeException e) {
    110                 System.out.println("File name: \n" + tokensFiles[i].getName());
    111                 String data = getResource(tokensFiles[i].getName());
    112                 System.out.println("Data: \n" + data);
    113                 System.out.println("Tokens:");
    114                 for (String token : tokens1) {
    115                     System.out.println(token);
    116                 }
    117                 fail("Cannot scan: " + tokensFiles[i]);
    118             }
    119         }
    120     }
    121
    122     public void testScanner() throws IOException {
    123         File[] files = getStreamsByExtension(".data", true);
    124         assertTrue("No test files found.", files.length > 0);
    125         for (File file : files) {
    126             List<String> tokens = new ArrayList<String>();
    127             InputStream input = new FileInputStream(file);
    128             StreamReader reader = new StreamReader(new UnicodeReader(input));
    129             Scanner scanner = new ScannerImpl(reader);
    130             try {
    131                 while (scanner.checkToken(new Token.ID[0])) {
    132                     Token token = scanner.getToken();
    133                     tokens.add(token.getClass().getName());
    134                 }
    135             } catch (RuntimeException e) {
    136                 System.out.println("File name: \n" + file.getName());
    137                 String data = getResource(file.getName());
    138                 System.out.println("Data: \n" + data);
    139                 System.out.println("Tokens:");
    140                 for (String token : tokens) {
    141                     System.out.println(token);
    142                 }
    143                 fail("Cannot scan: " + file + "; " + e.getMessage());
    144             } finally {
    145                 input.close();
    146             }
    147         }
    148     }
    149 }