Continue CLI batches after malformed JSON input

bin/jsonpointer

@@ -27,6 +27,8 @@ parser.add_argument('FILE', type=str, nargs='+',
                     help='Files for which the pointer should be resolved')
 parser.add_argument('--indent', type=int, default=None,
                     help='Indent output by n spaces')
+parser.add_argument('--continue-on-error', action='store_true',
+                    help='Continue with remaining files after JSON or text decoding errors')
 parser.add_argument('-v', '--version', action='version',
                     version='%(prog)s ' + jsonpointer.__version__)
 
@@ -77,8 +79,8 @@ def parse_pointer(args):
 def resolve_files():
     """ Resolve a JSON pointer on JSON files
 
-    Returns False if the pointer could not be resolved on at least one
-    file. """
+    Returns False if the pointer could not be resolved on at least one file,
+    or if --continue-on-error skipped a JSON or text decoding error. """
     args = parser.parse_args()
 
     ptr, files = parse_pointer(args)
@@ -90,7 +92,16 @@ def resolve_files():
         except argparse.ArgumentTypeError as e:
             parser.error(str(e))
 
-        doc = json.load(f)
+        try:
+            doc = json.load(f)
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            if not args.continue_on_error:
+                raise
+            print('Could not parse JSON in %r: %s' % (name, e),
+                  file=sys.stderr)
+            ok = False
+            continue
+
         try:
             result = jsonpointer.resolve_pointer(doc, ptr)
             print(json.dumps(result, indent=args.indent))

doc/commandline.rst

@@ -6,7 +6,8 @@ that can be used to resolve a JSON pointers on JSON files.
 
 The program has the following usage ::
 
-    usage: jsonpointer [-h] [-f POINTER_FILE | -p POINTER] [--indent INDENT] [-v]
+    usage: jsonpointer [-h] [-f POINTER_FILE | -p POINTER] [--indent INDENT]
+                       [--continue-on-error] [-v]
                        [POINTER] FILE [FILE ...]
 
     Resolve a JSON pointer on JSON files
@@ -23,6 +24,8 @@ The program has the following usage ::
       -p POINTER, --pointer POINTER
                             A JSON pointer expression
       --indent INDENT       Indent output by n spaces
+      --continue-on-error   Continue with remaining files after JSON or text
+                            decoding errors
       -v, --version         show program's version number and exit
 
 The pointer can be passed as the first positional argument, with ``-p``, or
@@ -33,6 +36,13 @@ could not be resolved on at least one file (the remaining files are still
 processed) and 2 on usage errors, such as an invalid pointer or a missing
 file.
 
+By default, invalid JSON or a text decoding error stops processing. With
+``--continue-on-error``, these errors are reported on standard error with
+the file name and decoder details, and the remaining files are processed.
+The exit status is 1 if any file has such an error, even if later files
+succeed. Missing files and other usage errors still stop processing with
+status 2.
+
 Example
 ^^^^^^^
 

tests.py

@@ -507,7 +507,7 @@ def _write(self, name, content):
             f.write(content)
 
     def _run(self, *args):
-        env = dict(os.environ, PYTHONPATH=self.ROOT)
+        env = dict(os.environ, PYTHONPATH=self.ROOT, PYTHONUTF8='1')
         return subprocess.run([sys.executable, self.SCRIPT] + list(args),
                               cwd=self.dir, env=env, capture_output=True,
                               text=True)
@@ -584,6 +584,92 @@ def test_unresolvable_pointer_in_one_file(self):
         self.assertIn('Could not resolve pointer', proc.stderr)
         self.assertEqual(proc.stdout, '1\n')
 
+    def test_invalid_json(self):
+        for content in ('', '{"a": }', '{"a": 1}\n{"a": 2}'):
+            with self.subTest(content=content):
+                self._write('invalid.json', content)
+                proc = self._run('--continue-on-error', '/a', 'invalid.json')
+                self.assertEqual(proc.returncode, 1)
+                self.assertEqual(proc.stdout, '')
+                self.assertIn("Could not parse JSON in 'invalid.json':",
+                              proc.stderr)
+                self.assertIn('line ', proc.stderr)
+                self.assertNotIn('Traceback', proc.stderr)
+
+    def test_invalid_json_in_one_file(self):
+        self._write('invalid.json', '{"a": }')
+        for index in range(3):
+            with self.subTest(index=index):
+                files = ['a.json', 'b.json']
+                files.insert(index, 'invalid.json')
+                proc = self._run('--continue-on-error', '/a', *files)
+                self.assertEqual(proc.returncode, 1)
+                self.assertEqual(proc.stdout, self.A_OUT + self.B_OUT)
+                self.assertIn("Could not parse JSON in 'invalid.json':",
+                              proc.stderr)
+                self.assertNotIn('Traceback', proc.stderr)
+
+    def test_invalid_json_and_unresolvable_pointer(self):
+        self._write('invalid.json', '{"a": }')
+        proc = self._run('--continue-on-error', '/b',
+                         'invalid.json', 'a.json', 'b.json')
+        self.assertEqual(proc.returncode, 1)
+        self.assertEqual(proc.stdout, '1\n')
+        self.assertIn("Could not parse JSON in 'invalid.json':", proc.stderr)
+        self.assertIn('Could not resolve pointer:', proc.stderr)
+        self.assertNotIn('Traceback', proc.stderr)
+
+    def test_invalid_json_stops_without_continue_on_error(self):
+        self._write('invalid.json', '{"a": }')
+        for index in range(3):
+            with self.subTest(index=index):
+                files = ['a.json', 'b.json']
+                files.insert(index, 'invalid.json')
+                proc = self._run('/a', *files)
+                self.assertEqual(proc.returncode, 1)
+                self.assertEqual(proc.stdout,
+                                 ''.join([self.A_OUT, self.B_OUT][:index]))
+                self.assertIn('JSONDecodeError', proc.stderr)
+
+    def test_invalid_encoding_in_one_file(self):
+        with open(os.path.join(self.dir, 'invalid.json'), 'wb') as f:
+            f.write(b'{"a":"\xff"}')
+        for index in range(3):
+            with self.subTest(index=index):
+                files = ['a.json', 'b.json']
+                files.insert(index, 'invalid.json')
+                proc = self._run('--continue-on-error', '/a', *files)
+                self.assertEqual(proc.returncode, 1)
+                self.assertEqual(proc.stdout, self.A_OUT + self.B_OUT)
+                self.assertIn("Could not parse JSON in 'invalid.json':",
+                              proc.stderr)
+                self.assertIn('decode', proc.stderr)
+                self.assertNotIn('Traceback', proc.stderr)
+
+    def test_invalid_encoding_stops_without_continue_on_error(self):
+        with open(os.path.join(self.dir, 'invalid.json'), 'wb') as f:
+            f.write(b'{"a":"\xff"}')
+        proc = self._run('/a', 'a.json', 'invalid.json', 'b.json')
+        self.assertEqual(proc.returncode, 1)
+        self.assertEqual(proc.stdout, self.A_OUT)
+        self.assertIn('UnicodeDecodeError', proc.stderr)
+
+    def test_continue_on_error_with_valid_files(self):
+        for pointer in (['/a'], ['-p', '/a'], ['-f', 'ptr.txt']):
+            with self.subTest(pointer=pointer):
+                proc = self._run('--continue-on-error', *pointer,
+                                 'a.json', 'b.json')
+                self.assertEqual(proc.returncode, 0, proc.stderr)
+                self.assertEqual(proc.stdout, self.A_OUT + self.B_OUT)
+                self.assertEqual(proc.stderr, '')
+
+    def test_continue_on_error_does_not_skip_missing_files(self):
+        proc = self._run('--continue-on-error', '/a',
+                         'a.json', 'missing.json', 'b.json')
+        self.assertEqual(proc.returncode, 2)
+        self.assertEqual(proc.stdout, self.A_OUT)
+        self.assertIn('missing.json', proc.stderr)
+
     def test_pointer_file_requires_value(self):
         proc = self._run('-f', '-p', '/a', 'a.json')
         self.assertEqual(proc.returncode, 2)