Commit

Author: StanFromIreland

Lib/test/test_codecencodings_jp.py

@@ -106,6 +106,27 @@ class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase):
         b"\x85Gℜ\x85Q = ⟨ሴ⟩"
     )
 
+    def test_null_terminator(self):
+        # see gh-101828
+        cases = (
+            "バルーンフルーツ",
+            "ライフアップキノコ",
+            "テスト",
+            "'Tis but a scratch!"
+        )
+        for case in cases:
+            with self.subTest(case=case):
+                encode_w_null = (case + "\0").encode(self.encoding)
+                encode_plus_null = case.encode(self.encoding) + "\0".encode(self.encoding)
+                self.assertTrue(encode_w_null.endswith(b'\x00'))
+                self.assertEqual(encode_w_null, encode_plus_null)
+
+                encode_w_null_2 = encode_w_null + encode_w_null
+                encode_plus_null_2 = encode_plus_null + encode_plus_null
+                self.assertEqual(encode_w_null_2.count(b'\x00'), 2)
+                self.assertEqual(encode_w_null_2, encode_plus_null_2)
+
+
 class Test_SJISX0213(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'shift_jisx0213'
     tstring = multibytecodec_support.load_teststring('shift_jisx0213')
@@ -121,6 +142,26 @@ class Test_SJISX0213(multibytecodec_support.TestBase, unittest.TestCase):
         "\xab\u211c\xbb = \u2329\u1234\u232a",
         b"\x85Gℜ\x85Q = ⟨ሴ⟩"
     )
+    def test_null_terminator(self):
+        # see gh-101828
+        cases = (
+            "バルーンフルーツ",
+            "ライフアップキノコ",
+            "テスト",
+            "'Tis but a scratch!"
+        )
+        for case in cases:
+            with self.subTest(case=case):
+                encode_w_null = (case + "\0").encode(self.encoding)
+                encode_plus_null = case.encode(self.encoding) + "\0".encode(self.encoding)
+                self.assertTrue(encode_w_null.endswith(b'\x00'))
+                self.assertEqual(encode_w_null, encode_plus_null)
+
+                encode_w_null_2 = encode_w_null + encode_w_null
+                encode_plus_null_2 = encode_plus_null + encode_plus_null
+                self.assertEqual(encode_w_null_2.count(b'\x00'), 2)
+                self.assertEqual(encode_w_null_2, encode_plus_null_2)
+
 
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst

@@ -0,0 +1,2 @@
+Fix ``'shift_jisx0213'`` and ``'shift_jis_2004'`` codecs truncating null char
+as it was treated as part of a multi-character sequence.

Modules/cjkcodecs/_codecs_jp.c

@@ -611,8 +611,19 @@ ENCODER(shift_jis_2004)
                             if (code == DBCINV)
                                 return 1;
                             }
-                            else
+                            else if (ch2 != 0) {
                                 insize = 2;
+                            }
+                            else {
+                                /* Don't consume null char as part of pair */
+                                code = find_pairencmap(
+                                    (ucs2_t)c, 0,
+                                    jisx0213_pair_encmap,
+                                    JISX0213_ENCPAIRS);
+                                if (code == DBCINV) {
+                                    return 1;
+                                }
+                            }
                         }
                     }
                 }