From 47d1037175fde162c8a4a521e274e7c6112bc55b Mon Sep 17 00:00:00 2001 From: Jesse Wilson Date: Sat, 12 Sep 2020 14:59:22 -0400 Subject: [PATCH] JsonValueSource: a brace-matching implementation of okio.Source This will read one JSON object and then stop. This is a part of https://github.com/square/moshi/issues/675 --- .../com/squareup/moshi/JsonValueSource.java | 208 ++++++++++++++++++ .../squareup/moshi/JsonValueSourceTest.java | 204 +++++++++++++++++ 2 files changed, 412 insertions(+) create mode 100644 moshi/src/main/java/com/squareup/moshi/JsonValueSource.java create mode 100644 moshi/src/test/java/com/squareup/moshi/JsonValueSourceTest.java diff --git a/moshi/src/main/java/com/squareup/moshi/JsonValueSource.java b/moshi/src/main/java/com/squareup/moshi/JsonValueSource.java new file mode 100644 index 0000000..0365d7c --- /dev/null +++ b/moshi/src/main/java/com/squareup/moshi/JsonValueSource.java @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2020 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.moshi; + +import java.io.EOFException; +import java.io.IOException; +import okio.Buffer; +import okio.BufferedSource; +import okio.ByteString; +import okio.Source; +import okio.Timeout; + +/** + * This source reads a prefix of another source as a JSON value and then terminates. It can read + * top-level arrays, objects, or strings only. + * + *

It implements {@linkplain JsonReader#setLenient lenient parsing} and has no mechanism to + * enforce strict parsing. If the input is not valid or lenient JSON the behavior of this source is + * unspecified. + */ +final class JsonValueSource implements Source { + private static final ByteString STATE_JSON = ByteString.encodeUtf8("[]{}\"'/#"); + private static final ByteString STATE_SINGLE_QUOTED = ByteString.encodeUtf8("'\\"); + private static final ByteString STATE_DOUBLE_QUOTED = ByteString.encodeUtf8("\"\\"); + private static final ByteString STATE_END_OF_LINE_COMMENT = ByteString.encodeUtf8("\r\n"); + private static final ByteString STATE_C_STYLE_COMMENT = ByteString.encodeUtf8("*"); + private static final ByteString STATE_END_OF_JSON = ByteString.EMPTY; + + private final BufferedSource source; + private final Buffer buffer; + + /** + * The state indicates what kind of data is readable at {@link #limit}. This also serves + * double-duty as the type of bytes we're interested in while in this state. + */ + private ByteString state = STATE_JSON; + + /** + * The level of nesting of arrays and objects. When the end of string, array, or object is + * reached, this should be compared against 0. If it is zero, then we've read a complete value and + * this source is exhausted. + */ + private int stackSize = 0; + + /** The number of bytes immediately returnable to the caller. */ + private long limit = 0; + + private boolean closed = false; + + public JsonValueSource(BufferedSource source) { + this.source = source; + this.buffer = source.getBuffer(); + } + + /** + * Advance {@link #limit} until it is at least {@code byteCount} or the JSON object is complete. + * + * @throws EOFException if the stream is exhausted before the JSON object completes. + */ + private void advanceLimit(long byteCount) throws IOException { + while (limit < byteCount) { + // If we've finished the JSON object, we're done. + if (state == STATE_END_OF_JSON) { + return; + } + + // If advancing requires more data in the buffer, grow it. + if (limit == buffer.size()) { + source.require(limit + 1L); + } + + // Find the next interesting character for the current state. If the buffer doesn't have one, + // then we can read the entire buffer. + long index = buffer.indexOfElement(state, limit); + if (index == -1L) { + limit = buffer.size(); + continue; + } + + byte b = buffer.getByte(index); + + if (state == STATE_JSON) { + switch (b) { + case '[': + case '{': + stackSize++; + limit = index + 1; + break; + + case ']': + case '}': + stackSize--; + if (stackSize == 0) state = STATE_END_OF_JSON; + limit = index + 1; + break; + + case '\"': + state = STATE_DOUBLE_QUOTED; + limit = index + 1; + break; + + case '\'': + state = STATE_SINGLE_QUOTED; + limit = index + 1; + break; + + case '/': + source.require(index + 2); + byte b2 = buffer.getByte(index + 1); + if (b2 == '/') { + state = STATE_END_OF_LINE_COMMENT; + limit = index + 2; + } else if (b2 == '*') { + state = STATE_C_STYLE_COMMENT; + limit = index + 2; + } else { + limit = index + 1; + } + break; + + case '#': + state = STATE_END_OF_LINE_COMMENT; + limit = index + 1; + break; + } + + } else if (state == STATE_SINGLE_QUOTED || state == STATE_DOUBLE_QUOTED) { + if (b == '\\') { + source.require(index + 2); + limit = index + 2; + } else { + state = (stackSize > 0) ? STATE_JSON : STATE_END_OF_JSON; + limit = index + 1; + } + + } else if (state == STATE_C_STYLE_COMMENT) { + source.require(index + 2); + if (buffer.getByte(index + 1) == '/') { + limit = index + 2; + state = STATE_JSON; + } else { + limit = index + 1; + } + + } else if (state == STATE_END_OF_LINE_COMMENT) { + limit = index + 1; + state = STATE_JSON; + + } else { + throw new AssertionError(); + } + } + } + + /** + * Discards any remaining JSON data in this source that was left behind after it was closed. It is + * an error to call {@link #read} after calling this method. + */ + public void discard() throws IOException { + closed = true; + while (state != STATE_END_OF_JSON) { + advanceLimit(8192); + source.skip(limit); + } + } + + @Override + public long read(Buffer sink, long byteCount) throws IOException { + if (closed) throw new IllegalStateException("closed"); + if (byteCount == 0) return 0L; + + advanceLimit(byteCount); + + if (limit == 0) { + if (state != STATE_END_OF_JSON) throw new AssertionError(); + return -1L; + } + + long result = Math.min(byteCount, limit); + sink.write(buffer, result); + limit -= result; + return result; + } + + @Override + public Timeout timeout() { + return source.timeout(); + } + + @Override + public void close() throws IOException { + // Note that this does not close the underlying source; that's the creator's responsibility. + closed = true; + } +} diff --git a/moshi/src/test/java/com/squareup/moshi/JsonValueSourceTest.java b/moshi/src/test/java/com/squareup/moshi/JsonValueSourceTest.java new file mode 100644 index 0000000..125e5e2 --- /dev/null +++ b/moshi/src/test/java/com/squareup/moshi/JsonValueSourceTest.java @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2020 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.moshi; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +import java.io.EOFException; +import java.io.IOException; +import okio.Buffer; +import org.junit.Test; + +public final class JsonValueSourceTest { + @Test + public void simpleValues() throws IOException { + assertThat(jsonPrefix("{\"hello\": \"world\"}, 1, 2, 3")).isEqualTo("{\"hello\": \"world\"}"); + assertThat(jsonPrefix("['hello', 'world'], 1, 2, 3")).isEqualTo("['hello', 'world']"); + assertThat(jsonPrefix("\"hello\", 1, 2, 3")).isEqualTo("\"hello\""); + } + + @Test + public void braceMatching() throws IOException { + assertThat(jsonPrefix("[{},{},[],[{}]],[]")).isEqualTo("[{},{},[],[{}]]"); + assertThat(jsonPrefix("[\"a\",{\"b\":{\"c\":{\"d\":[\"e\"]}}}],[]")) + .isEqualTo("[\"a\",{\"b\":{\"c\":{\"d\":[\"e\"]}}}]"); + } + + @Test + public void stringEscapes() throws IOException { + assertThat(jsonPrefix("[\"12\\u00334\"],[]")).isEqualTo("[\"12\\u00334\"]"); + assertThat(jsonPrefix("[\"12\\n34\"],[]")).isEqualTo("[\"12\\n34\"]"); + assertThat(jsonPrefix("[\"12\\\"34\"],[]")).isEqualTo("[\"12\\\"34\"]"); + assertThat(jsonPrefix("[\"12\\'34\"],[]")).isEqualTo("[\"12\\'34\"]"); + assertThat(jsonPrefix("[\"12\\\\34\"],[]")).isEqualTo("[\"12\\\\34\"]"); + assertThat(jsonPrefix("[\"12\\\\\"],[]")).isEqualTo("[\"12\\\\\"]"); + } + + @Test + public void bracesInStrings() throws IOException { + assertThat(jsonPrefix("[\"]\"],[]")).isEqualTo("[\"]\"]"); + assertThat(jsonPrefix("[\"\\]\"],[]")).isEqualTo("[\"\\]\"]"); + assertThat(jsonPrefix("[\"\\[\"],[]")).isEqualTo("[\"\\[\"]"); + } + + @Test + public void unterminatedString() throws IOException { + try { + jsonPrefix("{\"a\":\"b..."); + fail(); + } catch (EOFException expected) { + } + } + + @Test + public void unterminatedObject() throws IOException { + try { + jsonPrefix("{\"a\":\"b\",\"c\":"); + fail(); + } catch (EOFException expected) { + } + try { + jsonPrefix("{"); + fail(); + } catch (EOFException expected) { + } + } + + @Test + public void unterminatedArray() throws IOException { + try { + jsonPrefix("[\"a\",\"b\",\"c\","); + fail(); + } catch (EOFException expected) { + } + try { + jsonPrefix("["); + fail(); + } catch (EOFException expected) { + } + } + + @Test + public void lenientUnterminatedSingleQuotedString() throws IOException { + try { + jsonPrefix("{\"a\":'b..."); + fail(); + } catch (EOFException expected) { + } + } + + @Test + public void emptyStream() throws IOException { + try { + jsonPrefix(""); + fail(); + } catch (EOFException expected) { + } + try { + jsonPrefix(" "); + fail(); + } catch (EOFException expected) { + } + try { + jsonPrefix("/* comment */"); + fail(); + } catch (EOFException expected) { + } + } + + @Test + public void lenientSingleQuotedStrings() throws IOException { + assertThat(jsonPrefix("['hello', 'world'], 1, 2, 3")).isEqualTo("['hello', 'world']"); + assertThat(jsonPrefix("'abc\\'', 123")).isEqualTo("'abc\\''"); + } + + @Test + public void lenientCStyleComments() throws IOException { + assertThat(jsonPrefix("[\"a\"/* \"b\" */,\"c\"],[]")).isEqualTo("[\"a\"/* \"b\" */,\"c\"]"); + assertThat(jsonPrefix("[\"a\"/*]*/],[]")).isEqualTo("[\"a\"/*]*/]"); + assertThat(jsonPrefix("[\"a\"/**/],[]")).isEqualTo("[\"a\"/**/]"); + assertThat(jsonPrefix("[\"a\"/*/ /*/],[]")).isEqualTo("[\"a\"/*/ /*/]"); + assertThat(jsonPrefix("[\"a\"/*/ **/],[]")).isEqualTo("[\"a\"/*/ **/]"); + } + + @Test + public void lenientEndOfLineComments() throws IOException { + assertThat(jsonPrefix("[\"a\"// \"b\" \n,\"c\"],[]")).isEqualTo("[\"a\"// \"b\" \n,\"c\"]"); + assertThat(jsonPrefix("[\"a\"// \"b\" \r\n,\"c\"],[]")).isEqualTo("[\"a\"// \"b\" \r\n,\"c\"]"); + assertThat(jsonPrefix("[\"a\"// \"b\" \r,\"c\"],[]")).isEqualTo("[\"a\"// \"b\" \r,\"c\"]"); + assertThat(jsonPrefix("[\"a\"//]\r\n\"c\"],[]")).isEqualTo("[\"a\"//]\r\n\"c\"]"); + } + + @Test + public void lenientSlashInToken() throws IOException { + assertThat(jsonPrefix("{a/b:\"c\"},[]")).isEqualTo("{a/b:\"c\"}"); + } + + @Test + public void lenientUnterminatedEndOfLineComment() throws IOException { + try { + jsonPrefix("{\"a\",//}"); + fail(); + } catch (EOFException expected) { + } + } + + @Test + public void lenientUnterminatedCStyleComment() throws IOException { + try { + jsonPrefix("{\"a\",/* *"); + fail(); + } catch (EOFException expected) { + } + try { + jsonPrefix("{\"a\",/* **"); + fail(); + } catch (EOFException expected) { + } + try { + jsonPrefix("{\"a\",/* /**"); + fail(); + } catch (EOFException expected) { + } + } + + @Test + public void discard() throws IOException { + Buffer allData = new Buffer(); + allData.writeUtf8("{\"a\",\"b\",\"c\"},[\"d\", \"e\"]"); + + JsonValueSource jsonValueSource = new JsonValueSource(allData); + jsonValueSource.close(); + jsonValueSource.discard(); + + assertThat(allData.readUtf8()).isEqualTo(",[\"d\", \"e\"]"); + } + + private String jsonPrefix(String string) throws IOException { + Buffer allData = new Buffer(); + allData.writeUtf8(string); + + Buffer jsonPrefixBuffer = new Buffer(); + jsonPrefixBuffer.writeAll(new JsonValueSource(allData)); + + String result = jsonPrefixBuffer.readUtf8(); + String remainder = allData.readUtf8(); + assertThat(result + remainder).isEqualTo(string); + + return result; + } +}