mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 10:44:36 +08:00
Code Activity

csv: support non-ASCII Unicode dialect characters (#9367)

Browse Source 
* scripts: make check_redundant_patches.py executable

The redundant-test-patches pre-commit hook uses `language: script`, which
requires the executable bit, and fails on every commit touching Lib/test.

Assisted-by: Claude:claude-sonnet-5-5

* csv: support non-ASCII Unicode dialect characters

Store delimiter, quotechar and escapechar as `char` instead of `u8`, and
run the reader state machine and the writer over code points, so that a
single non-ASCII character such as `€` is accepted like in CPython. The
field size limit is now counted in characters rather than bytes.

Fixes #8408

Assisted-by: Claude:claude-sonnet-5-5
This commit is contained in:
Mathieu Dupuy authored and GitHub committed 2026-10-28 01:23:02 +09:00
1 parent b25ab5e76b
commit a463449e35
4 files changed
+129 -132
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Lib/test/test_csv.py
Vendored
-1
View File
@@ -672,7 +672,6 @@ class TestDialectRegistry(unittest.TestCase):
fileobj.seek(0) fileobj.seek(0)
self.assertEqual(fileobj.read(), expected) self.assertEqual(fileobj.read(), expected)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_dialect_apply(self): def test_dialect_apply(self):
class testA(csv.excel): class testA(csv.excel):
delimiter = "\t" delimiter = "\t"
crates/stdlib/src/csv.rs
+105 -131
View File
@@ -73,13 +73,17 @@ mod _csv {
) )
} }
fn utf8_data<'a>(s: &'a PyStr, vm: &VirtualMachine) -> PyResult<&'a str> {
core::str::from_utf8(s.as_bytes()).map_err(|e| new_not_utf8_error(vm, s.as_bytes(), e))
}
#[pyattr] #[pyattr]
#[pyclass(module = "_csv", name = "Dialect")] #[pyclass(module = "_csv", name = "Dialect")]
#[derive(Debug, PyPayload, Clone)] #[derive(Debug, PyPayload, Clone)]
struct PyDialect { struct PyDialect {
delimiter: u8, delimiter: char,
quotechar: Option<u8>, quotechar: Option<char>,
escapechar: Option<u8>, escapechar: Option<char>,
doublequote: bool, doublequote: bool,
skipinitialspace: bool, skipinitialspace: bool,
lineterminator: String, lineterminator: String,
@@ -99,12 +103,12 @@ mod _csv {
impl PyDialect { impl PyDialect {
#[pygetset] #[pygetset]
fn delimiter(zelf: &Py<Self>, vm: &VirtualMachine) -> PyRef<PyStr> { fn delimiter(zelf: &Py<Self>, vm: &VirtualMachine) -> PyRef<PyStr> {
vm.ctx.new_str(format!("{}", zelf.delimiter as char)) vm.ctx.new_str(zelf.delimiter)
} }
#[pygetset] #[pygetset]
fn quotechar(zelf: &Py<Self>, vm: &VirtualMachine) -> Option<PyRef<PyStr>> { fn quotechar(zelf: &Py<Self>, vm: &VirtualMachine) -> Option<PyRef<PyStr>> {
Some(vm.ctx.new_str(format!("{}", zelf.quotechar? as char))) Some(vm.ctx.new_str(zelf.quotechar?))
} }
#[pygetset] #[pygetset]
@@ -129,7 +133,7 @@ mod _csv {
#[pygetset] #[pygetset]
fn escapechar(zelf: &Py<Self>, vm: &VirtualMachine) -> Option<PyRef<PyStr>> { fn escapechar(zelf: &Py<Self>, vm: &VirtualMachine) -> Option<PyRef<PyStr>> {
Some(vm.ctx.new_str(format!("{}", zelf.escapechar? as char))) Some(vm.ctx.new_str(zelf.escapechar?))
} }
#[pygetset(name = "strict")] #[pygetset(name = "strict")]
@@ -143,7 +147,7 @@ mod _csv {
obj: &PyObject, obj: &PyObject,
name: &str, name: &str,
allow_none: bool, allow_none: bool,
) -> PyResult<Option<u8>> { ) -> PyResult<Option<char>> {
if allow_none && vm.is_none(obj) { if allow_none && vm.is_none(obj) {
return Ok(None); return Ok(None);
} }
@@ -174,13 +178,13 @@ mod _csv {
fn parse_single_char( fn parse_single_char(
s: &Py<PyStr>, s: &Py<PyStr>,
error: impl Fn(usize) -> PyBaseExceptionRef, error: impl Fn(usize) -> PyBaseExceptionRef,
) -> PyResult<u8> { ) -> PyResult<char> {
let ch = s let ch = s
.as_wtf8() .as_wtf8()
.code_points() .code_points()
.exactly_one() .exactly_one()
.map_err(|_| error(s.char_len()))?; .map_err(|_| error(s.char_len()))?;
u8::try_from(ch.to_u32()).map_err(|_| error(s.char_len())) ch.to_char().ok_or_else(|| error(s.char_len()))
} }
#[derive(FromArgs)] #[derive(FromArgs)]
@@ -412,8 +416,8 @@ mod _csv {
} }
fn validate_dialect(vm: &VirtualMachine, dialect: &PyDialect) -> PyResult<()> { fn validate_dialect(vm: &VirtualMachine, dialect: &PyDialect) -> PyResult<()> {
let special = |name: &str, value: u8| { let special = |name: &str, value: char| {
if matches!(value, b'\r' | b'\n') { if matches!(value, '\r' | '\n') {
Err(vm.new_value_error(format!( Err(vm.new_value_error(format!(
"{name} must be a single character, not a line break" "{name} must be a single character, not a line break"
))) )))
@@ -431,14 +435,14 @@ mod _csv {
} }
if dialect.skipinitialspace if dialect.skipinitialspace
&& (matches!(dialect.escapechar, Some(b' ')) || matches!(dialect.quotechar, Some(b' '))) && (matches!(dialect.escapechar, Some(' ')) || matches!(dialect.quotechar, Some(' ')))
{ {
return Err(vm.new_value_error( return Err(vm.new_value_error(
"escapechar or quotechar cannot be a space when skipinitialspace is enabled", "escapechar or quotechar cannot be a space when skipinitialspace is enabled",
)); ));
} }
let values: [(&str, Option<u8>); 3] = [ let values: [(&str, Option<char>); 3] = [
("delimiter", Some(dialect.delimiter)), ("delimiter", Some(dialect.delimiter)),
("quotechar", dialect.quotechar), ("quotechar", dialect.quotechar),
("escapechar", dialect.escapechar), ("escapechar", dialect.escapechar),
@@ -451,12 +455,7 @@ mod _csv {
))); )));
} }
} }
if left.is_some_and(|value| { if left.is_some_and(|value| dialect.lineterminator.contains(value)) {
dialect
.lineterminator
.chars()
.any(|character| character == value as char)
}) {
return Err(vm.new_value_error(format!( return Err(vm.new_value_error(format!(
"{left_name} and lineterminator cannot be the same" "{left_name} and lineterminator cannot be the same"
))); )));
@@ -498,7 +497,7 @@ mod _csv {
let delimiter = match &self.delimiter { let delimiter = match &self.delimiter {
Some(obj) => parse_char(vm, obj, "delimiter", false)?.unwrap(), Some(obj) => parse_char(vm, obj, "delimiter", false)?.unwrap(),
None => b',', None => ',',
}; };
let doublequote = match &self.doublequote { let doublequote = match &self.doublequote {
Some(obj) => obj.try_to_bool(vm)?, Some(obj) => obj.try_to_bool(vm)?,
@@ -526,7 +525,7 @@ mod _csv {
}; };
let quotechar = match &self.quotechar { let quotechar = match &self.quotechar {
Some(obj) => parse_char(vm, obj, "quotechar", true)?, Some(obj) => parse_char(vm, obj, "quotechar", true)?,
None => Some(b'"'), None => Some('"'),
}; };
let quoting = match &self.quoting { let quoting = match &self.quoting {
Some(obj) => { Some(obj) => {
@@ -628,7 +627,7 @@ mod _csv {
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ParserInput { enum ParserInput {
Byte(u8), Char(char),
Eol, Eol,
} }
@@ -637,7 +636,8 @@ mod _csv {
struct CsvParser { struct CsvParser {
state: ParserState, state: ParserState,
fields: Vec<PyObjectRef>, fields: Vec<PyObjectRef>,
field: Vec<u8>, field: String,
field_len: usize,
unquoted_field: bool, unquoted_field: bool,
field_limit: isize, field_limit: isize,
} }
@@ -647,7 +647,8 @@ mod _csv {
Self { Self {
state: ParserState::StartRecord, state: ParserState::StartRecord,
fields: Vec::new(), fields: Vec::new(),
field: Vec::new(), field: String::new(),
field_len: 0,
unquoted_field: false, unquoted_field: false,
field_limit, field_limit,
} }
@@ -657,14 +658,15 @@ mod _csv {
PyIterReturn::Return(vm.ctx.new_list(self.fields).into()) PyIterReturn::Return(vm.ctx.new_list(self.fields).into())
} }
fn add_byte(&mut self, byte: u8, vm: &VirtualMachine) -> PyResult<()> { fn add_char(&mut self, ch: char, vm: &VirtualMachine) -> PyResult<()> {
if self.field_limit < 0 || self.field.len() >= self.field_limit as usize { if self.field_limit < 0 || self.field_len >= self.field_limit as usize {
return Err(new_csv_error( return Err(new_csv_error(
vm, vm,
format!("field larger than field limit ({})", self.field_limit), format!("field larger than field limit ({})", self.field_limit),
)); ));
} }
self.field.push(byte); self.field.push(ch);
self.field_len += 1;
Ok(()) Ok(())
} }
@@ -675,9 +677,7 @@ mod _csv {
{ {
vm.ctx.none() vm.ctx.none()
} else { } else {
let value = core::str::from_utf8(&self.field) let field: PyObjectRef = vm.ctx.new_str(self.field.as_str()).into();
.map_err(|e| new_not_utf8_error(vm, &self.field, e))?;
let field: PyObjectRef = vm.ctx.new_str(value).into();
if self.unquoted_field if self.unquoted_field
&& !self.field.is_empty() && !self.field.is_empty()
&& matches!(quoting, QuoteStyle::Nonnumeric | QuoteStyle::Strings) && matches!(quoting, QuoteStyle::Nonnumeric | QuoteStyle::Strings)
@@ -689,6 +689,7 @@ mod _csv {
}; };
self.fields.push(field); self.fields.push(field);
self.field.clear(); self.field.clear();
self.field_len = 0;
Ok(()) Ok(())
} }
@@ -701,7 +702,7 @@ mod _csv {
match self.state { match self.state {
ParserState::StartRecord => match input { ParserState::StartRecord => match input {
ParserInput::Eol => {} ParserInput::Eol => {}
ParserInput::Byte(b'\r' | b'\n') => self.state = ParserState::EatCrnl, ParserInput::Char('\r' | '\n') => self.state = ParserState::EatCrnl,
_ => { _ => {
self.state = ParserState::StartField; self.state = ParserState::StartField;
return self.process_parser_input(input, dialect, vm); return self.process_parser_input(input, dialect, vm);
@@ -710,41 +711,41 @@ mod _csv {
ParserState::StartField => { ParserState::StartField => {
self.unquoted_field = true; self.unquoted_field = true;
match input { match input {
ParserInput::Eol | ParserInput::Byte(b'\r' | b'\n') => { ParserInput::Eol | ParserInput::Char('\r' | '\n') => {
self.save_field(dialect.quoting, vm)?; self.save_field(dialect.quoting, vm)?;
self.state = state_after_record_end(input); self.state = state_after_record_end(input);
} }
ParserInput::Byte(byte) ParserInput::Char(ch)
if dialect.quoting != QuoteStyle::None if dialect.quoting != QuoteStyle::None
&& dialect.quotechar == Some(byte) => && dialect.quotechar == Some(ch) =>
{ {
self.unquoted_field = false; self.unquoted_field = false;
self.state = ParserState::InQuotedField; self.state = ParserState::InQuotedField;
} }
ParserInput::Byte(byte) if dialect.escapechar == Some(byte) => { ParserInput::Char(ch) if dialect.escapechar == Some(ch) => {
self.state = ParserState::EscapedChar; self.state = ParserState::EscapedChar;
} }
ParserInput::Byte(b' ') if dialect.skipinitialspace => {} ParserInput::Char(' ') if dialect.skipinitialspace => {}
ParserInput::Byte(byte) if byte == dialect.delimiter => { ParserInput::Char(ch) if ch == dialect.delimiter => {
self.save_field(dialect.quoting, vm)?; self.save_field(dialect.quoting, vm)?;
} }
ParserInput::Byte(byte) => { ParserInput::Char(ch) => {
self.add_byte(byte, vm)?; self.add_char(ch, vm)?;
self.state = ParserState::InField; self.state = ParserState::InField;
} }
} }
} }
ParserState::EscapedChar => match input { ParserState::EscapedChar => match input {
ParserInput::Byte(byte @ (b'\r' | b'\n')) => { ParserInput::Char(ch @ ('\r' | '\n')) => {
self.add_byte(byte, vm)?; self.add_char(ch, vm)?;
self.state = ParserState::AfterEscapedCrnl; self.state = ParserState::AfterEscapedCrnl;
} }
ParserInput::Eol => { ParserInput::Eol => {
self.add_byte(b'\n', vm)?; self.add_char('\n', vm)?;
self.state = ParserState::InField; self.state = ParserState::InField;
} }
ParserInput::Byte(byte) => { ParserInput::Char(ch) => {
self.add_byte(byte, vm)?; self.add_char(ch, vm)?;
self.state = ParserState::InField; self.state = ParserState::InField;
} }
}, },
@@ -755,27 +756,26 @@ mod _csv {
} }
} }
ParserState::InField => match input { ParserState::InField => match input {
ParserInput::Eol | ParserInput::Byte(b'\r' | b'\n') => { ParserInput::Eol | ParserInput::Char('\r' | '\n') => {
self.save_field(dialect.quoting, vm)?; self.save_field(dialect.quoting, vm)?;
self.state = state_after_record_end(input); self.state = state_after_record_end(input);
} }
ParserInput::Byte(byte) if dialect.escapechar == Some(byte) => { ParserInput::Char(ch) if dialect.escapechar == Some(ch) => {
self.state = ParserState::EscapedChar; self.state = ParserState::EscapedChar;
} }
ParserInput::Byte(byte) if byte == dialect.delimiter => { ParserInput::Char(ch) if ch == dialect.delimiter => {
self.save_field(dialect.quoting, vm)?; self.save_field(dialect.quoting, vm)?;
self.state = ParserState::StartField; self.state = ParserState::StartField;
} }
ParserInput::Byte(byte) => self.add_byte(byte, vm)?, ParserInput::Char(ch) => self.add_char(ch, vm)?,
}, },
ParserState::InQuotedField => match input { ParserState::InQuotedField => match input {
ParserInput::Eol => {} ParserInput::Eol => {}
ParserInput::Byte(byte) if dialect.escapechar == Some(byte) => { ParserInput::Char(ch) if dialect.escapechar == Some(ch) => {
self.state = ParserState::EscapeInQuotedField; self.state = ParserState::EscapeInQuotedField;
} }
ParserInput::Byte(byte) ParserInput::Char(ch)
if dialect.quoting != QuoteStyle::None if dialect.quoting != QuoteStyle::None && dialect.quotechar == Some(ch) =>
&& dialect.quotechar == Some(byte) =>
{ {
self.state = if dialect.doublequote { self.state = if dialect.doublequote {
ParserState::QuoteInQuotedField ParserState::QuoteInQuotedField
@@ -783,51 +783,50 @@ mod _csv {
ParserState::InField ParserState::InField
}; };
} }
ParserInput::Byte(byte) => self.add_byte(byte, vm)?, ParserInput::Char(ch) => self.add_char(ch, vm)?,
}, },
ParserState::EscapeInQuotedField => { ParserState::EscapeInQuotedField => {
let byte = match input { let ch = match input {
ParserInput::Eol => b'\n', ParserInput::Eol => '\n',
ParserInput::Byte(byte) => byte, ParserInput::Char(ch) => ch,
}; };
self.add_byte(byte, vm)?; self.add_char(ch, vm)?;
self.state = ParserState::InQuotedField; self.state = ParserState::InQuotedField;
} }
ParserState::QuoteInQuotedField => match input { ParserState::QuoteInQuotedField => match input {
ParserInput::Byte(byte) ParserInput::Char(ch)
if dialect.quoting != QuoteStyle::None if dialect.quoting != QuoteStyle::None && dialect.quotechar == Some(ch) =>
&& dialect.quotechar == Some(byte) =>
{ {
self.add_byte(byte, vm)?; self.add_char(ch, vm)?;
self.state = ParserState::InQuotedField; self.state = ParserState::InQuotedField;
} }
ParserInput::Byte(byte) if byte == dialect.delimiter => { ParserInput::Char(ch) if ch == dialect.delimiter => {
self.save_field(dialect.quoting, vm)?; self.save_field(dialect.quoting, vm)?;
self.state = ParserState::StartField; self.state = ParserState::StartField;
} }
ParserInput::Eol | ParserInput::Byte(b'\r' | b'\n') => { ParserInput::Eol | ParserInput::Char('\r' | '\n') => {
self.save_field(dialect.quoting, vm)?; self.save_field(dialect.quoting, vm)?;
self.state = state_after_record_end(input); self.state = state_after_record_end(input);
} }
ParserInput::Byte(byte) if !dialect.strict => { ParserInput::Char(ch) if !dialect.strict => {
self.add_byte(byte, vm)?; self.add_char(ch, vm)?;
self.state = ParserState::InField; self.state = ParserState::InField;
} }
ParserInput::Byte(_) => { ParserInput::Char(_) => {
return Err(new_csv_error( return Err(new_csv_error(
vm, vm,
format!( format!(
"'{}' expected after '{}'", "'{}' expected after '{}'",
dialect.delimiter as char, dialect.delimiter,
dialect.quotechar.unwrap_or_default() as char, dialect.quotechar.unwrap_or_default(),
), ),
)); ));
} }
}, },
ParserState::EatCrnl => match input { ParserState::EatCrnl => match input {
ParserInput::Byte(b'\r' | b'\n') => {} ParserInput::Char('\r' | '\n') => {}
ParserInput::Eol => self.state = ParserState::StartRecord, ParserInput::Eol => self.state = ParserState::StartRecord,
ParserInput::Byte(_) => { ParserInput::Char(_) => {
return Err(new_csv_error( return Err(new_csv_error(
vm, vm,
concat!( concat!(
@@ -906,9 +905,9 @@ mod _csv {
zelf.state.lock().line_num += 1; zelf.state.lock().line_num += 1;
parser.field_limit = *GLOBAL_FIELD_LIMIT.lock(); parser.field_limit = *GLOBAL_FIELD_LIMIT.lock();
for &byte in string.as_bytes() { for ch in utf8_data(&string, vm)?.chars() {
parser.process_parser_input( parser.process_parser_input(
ParserInput::Byte(byte), ParserInput::Char(ch),
&zelf.dialect, &zelf.dialect,
vm, vm,
)?; )?;
@@ -945,8 +944,8 @@ mod _csv {
} }
fn write_quoted_field( fn write_quoted_field(
output: &mut Vec<u8>, output: &mut String,
data: &[u8], data: &str,
dialect: &PyDialect, dialect: &PyDialect,
vm: &VirtualMachine, vm: &VirtualMachine,
) -> PyResult<()> { ) -> PyResult<()> {
@@ -954,22 +953,22 @@ mod _csv {
.quotechar .quotechar
.ok_or_else(|| vm.new_type_error("quotechar must be set if quoting enabled"))?; .ok_or_else(|| vm.new_type_error("quotechar must be set if quoting enabled"))?;
output.push(quotechar); output.push(quotechar);
for &byte in data { for ch in data.chars() {
if byte == quotechar { if ch == quotechar {
if dialect.doublequote { if dialect.doublequote {
output.push(quotechar); output.push(quotechar);
output.push(quotechar); output.push(quotechar);
} else if let Some(escapechar) = dialect.escapechar { } else if let Some(escapechar) = dialect.escapechar {
output.push(escapechar); output.push(escapechar);
output.push(byte); output.push(ch);
} else { } else {
return Err(new_csv_error(vm, "need to escape, but no escapechar set")); return Err(new_csv_error(vm, "need to escape, but no escapechar set"));
} }
} else { } else {
if dialect.escapechar == Some(byte) { if dialect.escapechar == Some(ch) {
output.push(byte); output.push(ch);
} }
output.push(byte); output.push(ch);
} }
} }
output.push(quotechar); output.push(quotechar);
@@ -977,61 +976,38 @@ mod _csv {
} }
fn write_unquoted_field( fn write_unquoted_field(
output: &mut Vec<u8>, output: &mut String,
data: &[u8], data: &str,
dialect: &PyDialect, dialect: &PyDialect,
vm: &VirtualMachine, vm: &VirtualMachine,
) -> PyResult<()> { ) -> PyResult<()> {
let mut data = data; for ch in data.chars() {
while let Some((&byte, rest)) = data.split_first() { if char_needs_escape(ch, dialect) {
if field_needs_escape(data, dialect) {
let escapechar = dialect let escapechar = dialect
.escapechar .escapechar
.ok_or_else(|| new_csv_error(vm, "need to escape, but no escapechar set"))?; .ok_or_else(|| new_csv_error(vm, "need to escape, but no escapechar set"))?;
output.push(escapechar); output.push(escapechar);
} }
output.push(byte); output.push(ch);
data = rest;
} }
Ok(()) Ok(())
} }
fn data_contains_lineterminator_char(data: &[u8], dialect: &PyDialect) -> bool { fn field_needs_quotes(data: &str, dialect: &PyDialect) -> bool {
dialect.lineterminator.chars().any(|character| { data.chars().any(|ch| {
let mut encoded = [0; 4]; ch == dialect.delimiter
let character = character.encode_utf8(&mut encoded).as_bytes(); || (dialect.doublequote && dialect.quotechar == Some(ch))
data.windows(character.len()) || matches!(ch, '\r' | '\n')
.any(|window| window == character) || dialect.lineterminator.contains(ch)
}) })
} }
fn data_starts_with_lineterminator_char(data: &[u8], dialect: &PyDialect) -> bool { fn char_needs_escape(ch: char, dialect: &PyDialect) -> bool {
dialect.lineterminator.chars().any(|character| { ch == dialect.delimiter
let mut encoded = [0; 4]; || dialect.quotechar == Some(ch)
let character = character.encode_utf8(&mut encoded).as_bytes(); || dialect.escapechar == Some(ch)
data.starts_with(character) || matches!(ch, '\r' | '\n')
}) || dialect.lineterminator.contains(ch)
}
fn field_needs_quotes(data: &[u8], dialect: &PyDialect) -> bool {
data.iter().any(|&byte| {
byte == dialect.delimiter
|| (dialect.doublequote && dialect.quotechar == Some(byte))
|| matches!(byte, b'\r' | b'\n')
}) || data_contains_lineterminator_char(data, dialect)
}
fn field_needs_escape(data: &[u8], dialect: &PyDialect) -> bool {
let byte = data[0];
byte == dialect.delimiter
|| dialect.quotechar == Some(byte)
|| dialect.escapechar == Some(byte)
|| matches!(byte, b'\r' | b'\n')
|| data_starts_with_lineterminator_char(data, dialect)
}
fn write_lineterminator(output: &mut Vec<u8>, terminator: &str) {
output.extend_from_slice(terminator.as_bytes());
} }
#[pyclass(flags(DISALLOW_INSTANTIATION))] #[pyclass(flags(DISALLOW_INSTANTIATION))]
@@ -1053,7 +1029,7 @@ mod _csv {
})?; })?;
let fields = row.iter(vm)?.collect::<PyResult<Vec<_>>>()?; let fields = row.iter(vm)?.collect::<PyResult<Vec<_>>>()?;
let single_field = fields.len() == 1; let single_field = fields.len() == 1;
let mut output = Vec::new(); let mut output = String::new();
for (index, field) in fields.into_iter().enumerate() { for (index, field) in fields.into_iter().enumerate() {
if index > 0 { if index > 0 {
@@ -1061,12 +1037,12 @@ mod _csv {
} }
let stringified; let stringified;
let (data, is_str, is_none): (&[u8], bool, bool) = match_class!(match field { let (data, is_str, is_none): (&str, bool, bool) = match_class!(match field {
ref s @ PyStr => (s.as_bytes(), true, false), ref s @ PyStr => (utf8_data(s, vm)?, true, false),
crate::builtins::PyNone => (b"", false, true), crate::builtins::PyNone => ("", false, true),
ref obj => { ref obj => {
stringified = obj.str(vm)?; stringified = obj.str(vm)?;
(stringified.as_bytes(), false, false) (utf8_data(&stringified, vm)?, false, false)
} }
}); });
@@ -1086,7 +1062,7 @@ mod _csv {
} }
if data.is_empty() if data.is_empty()
&& zelf.dialect.delimiter == b' ' && zelf.dialect.delimiter == ' '
&& zelf.dialect.skipinitialspace && zelf.dialect.skipinitialspace
&& (zelf.dialect.quoting == QuoteStyle::None && (zelf.dialect.quoting == QuoteStyle::None
|| (is_none || (is_none
@@ -1111,7 +1087,7 @@ mod _csv {
if zelf.dialect.quoting != QuoteStyle::None if zelf.dialect.quoting != QuoteStyle::None
&& ((data.is_empty() && ((data.is_empty()
&& zelf.dialect.delimiter == b' ' && zelf.dialect.delimiter == ' '
&& zelf.dialect.skipinitialspace) && zelf.dialect.skipinitialspace)
|| (single_field && data.is_empty()) || (single_field && data.is_empty())
|| (!should_quote && field_needs_quotes(data, &zelf.dialect))) || (!should_quote && field_needs_quotes(data, &zelf.dialect)))
@@ -1126,10 +1102,8 @@ mod _csv {
} }
} }
write_lineterminator(&mut output, &zelf.dialect.lineterminator); output.push_str(&zelf.dialect.lineterminator);
let s = zelf.write.call((output,), vm)
core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;
zelf.write.call((s,), vm)
} }
#[pymethod] #[pymethod]
extra_tests/snippets/stdlib_csv.py
+24
View File
@@ -639,3 +639,27 @@ def test_field_size_limit_types():
test_field_size_limit_types() test_field_size_limit_types()
def test_non_ascii_dialect_chars():
for kwargs in (
{"delimiter": "€"},
{"quotechar": "€"},
{"escapechar": "€", "quoting": csv.QUOTE_NONE},
):
out = io.StringIO()
csv.writer(out, **kwargs).writerow(["a€b", 'é"x', "plain"])
out.seek(0)
assert list(csv.reader(out, **kwargs)) == [["a€b", 'é"x', "plain"]]
assert csv.writer(io.StringIO(), delimiter="€").dialect.delimiter == "€"
assert list(csv.reader(['a€b€"c€d"'], delimiter="€")) == [["a", "b", "c€d"]]
for bad in ("", "€€"):
with assert_raises(TypeError):
csv.writer(io.StringIO(), delimiter=bad)
with assert_raises(ValueError):
csv.writer(io.StringIO(), delimiter="€", quotechar="€")
test_non_ascii_dialect_chars()
scripts/check_redundant_patches.py
Regular → Executable
View File
File mode changed.