When reading data from a file into a str, check if it's utf-8

jepler · jepler · commit 06f22cb59c49 · 2022-08-12T08:25:27.000-05:00
Otherwise, weird stuff can happen down the line when it is print()ed,
especially as it can break the webrepl of circuitpython.
diff --git a/py/stream.c b/py/stream.c
@@ -31,6 +31,7 @@
 #include "py/objstr.h"
 #include "py/stream.h"
 #include "py/runtime.h"
+#include "py/unicode.h"
 #include "supervisor/shared/translate/translate.h"
 
 // This file defines generic Python stream read/write methods which
@@ -43,6 +44,13 @@ STATIC mp_obj_t stream_readall(mp_obj_t self_in);
 
 #define STREAM_CONTENT_TYPE(stream) (((stream)->is_text) ? &mp_type_str : &mp_type_bytes)
 
+static mp_obj_t mp_obj_new_str_from_vstr_check(const mp_obj_type_t *type, vstr_t *vstr) {
+    if (type == &mp_type_str && !utf8_check((void *)vstr->buf, vstr->len)) {
+        mp_raise_msg(&mp_type_UnicodeError, NULL);
+    }
+    return mp_obj_new_str_from_vstr(type, vstr);
+}
+
 // Returns error condition in *errcode, if non-zero, return value is number of bytes written
 // before error condition occurred. If *errcode == 0, returns total bytes written (which will
 // be equal to input size).
@@ -201,8 +209,7 @@ STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte fl
                 }
             }
         }
-
-        return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
+        return mp_obj_new_str_from_vstr_check(&mp_type_str, &vstr);
     }
     #endif
 
@@ -223,7 +230,7 @@ STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte fl
         mp_raise_OSError(error);
     } else {
         vstr.len = out_sz;
-        return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
+        return mp_obj_new_str_from_vstr_check(STREAM_CONTENT_TYPE(stream_p), &vstr);
     }
 }
 
@@ -364,7 +371,7 @@ STATIC mp_obj_t stream_readall(mp_obj_t self_in) {
     }
 
     vstr.len = total_size;
-    return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
+    return mp_obj_new_str_from_vstr_check(STREAM_CONTENT_TYPE(stream_p), &vstr);
 }
 
 // Unbuffered, inefficient implementation of readline() for raw I/O files.
@@ -417,7 +424,7 @@ STATIC mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args)
         }
     }
 
-    return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
+    return mp_obj_new_str_from_vstr_check(STREAM_CONTENT_TYPE(stream_p), &vstr);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj, 1, 2, stream_unbuffered_readline);
 

Original file line number	Diff line number	Diff line change
`@@ -31,6 +31,7 @@`
`31`	`31`	`#include "py/objstr.h"`
`32`	`32`	`#include "py/stream.h"`
`33`	`33`	`#include "py/runtime.h"`
	`34`	`+#include "py/unicode.h"`
`34`	`35`	`#include "supervisor/shared/translate/translate.h"`
`35`	`36`
`36`	`37`	`// This file defines generic Python stream read/write methods which`
`@@ -43,6 +44,13 @@ STATIC mp_obj_t stream_readall(mp_obj_t self_in);`
`43`	`44`
`44`	`45`	`#define STREAM_CONTENT_TYPE(stream) (((stream)->is_text) ? &mp_type_str : &mp_type_bytes)`
`45`	`46`
	`47`	`+static mp_obj_t mp_obj_new_str_from_vstr_check(const mp_obj_type_t type, vstr_t vstr) {`
	`48`	`+ if (type == &mp_type_str && !utf8_check((void *)vstr->buf, vstr->len)) {`
	`49`	`+ mp_raise_msg(&mp_type_UnicodeError, NULL);`
	`50`	`+ }`
	`51`	`+ return mp_obj_new_str_from_vstr(type, vstr);`
	`52`	`+}`
	`53`	`+`
`46`	`54`	`// Returns error condition in *errcode, if non-zero, return value is number of bytes written`
`47`	`55`	`// before error condition occurred. If *errcode == 0, returns total bytes written (which will`
`48`	`56`	`// be equal to input size).`
`@@ -201,8 +209,7 @@ STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte fl`
`201`	`209`	`}`
`202`	`210`	`}`
`203`	`211`	`}`
`204`		`-`
`205`		`- return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);`
	`212`	`+ return mp_obj_new_str_from_vstr_check(&mp_type_str, &vstr);`
`206`	`213`	`}`
`207`	`214`	`#endif`
`208`	`215`
`@@ -223,7 +230,7 @@ STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte fl`
`223`	`230`	`mp_raise_OSError(error);`
`224`	`231`	`} else {`
`225`	`232`	`vstr.len = out_sz;`
`226`		`- return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);`
	`233`	`+ return mp_obj_new_str_from_vstr_check(STREAM_CONTENT_TYPE(stream_p), &vstr);`
`227`	`234`	`}`
`228`	`235`	`}`
`229`	`236`
`@@ -364,7 +371,7 @@ STATIC mp_obj_t stream_readall(mp_obj_t self_in) {`
`364`	`371`	`}`
`365`	`372`
`366`	`373`	`vstr.len = total_size;`
`367`		`- return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);`
	`374`	`+ return mp_obj_new_str_from_vstr_check(STREAM_CONTENT_TYPE(stream_p), &vstr);`
`368`	`375`	`}`
`369`	`376`
`370`	`377`	`// Unbuffered, inefficient implementation of readline() for raw I/O files.`
`@@ -417,7 +424,7 @@ STATIC mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args)`
`417`	`424`	`}`
`418`	`425`	`}`
`419`	`426`
`420`		`- return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);`
	`427`	`+ return mp_obj_new_str_from_vstr_check(STREAM_CONTENT_TYPE(stream_p), &vstr);`
`421`	`428`	`}`
`422`	`429`	`MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj, 1, 2, stream_unbuffered_readline);`
`423`	`430`