r21848 - in /branches/upstream/libtext-csv-perl/current: Changes MANIFEST META.yml README files/ files/utf8.csv lib/Text/CSV.pm lib/Text/CSV_PP.pm t/50_utf8.t t/70_rt.t t/75_hashref.t t/80_diag.t
gregoa at users.alioth.debian.org
gregoa at users.alioth.debian.org
Wed Jun 18 17:23:59 UTC 2008
Author: gregoa
Date: Wed Jun 18 17:23:59 2008
New Revision: 21848
URL: http://svn.debian.org/wsvn/?sc=1&rev=21848
Log:
[svn-upgrade] Integrating new upstream version, libtext-csv-perl (1.06)
Added:
branches/upstream/libtext-csv-perl/current/files/
branches/upstream/libtext-csv-perl/current/files/utf8.csv
Modified:
branches/upstream/libtext-csv-perl/current/Changes
branches/upstream/libtext-csv-perl/current/MANIFEST
branches/upstream/libtext-csv-perl/current/META.yml
branches/upstream/libtext-csv-perl/current/README
branches/upstream/libtext-csv-perl/current/lib/Text/CSV.pm
branches/upstream/libtext-csv-perl/current/lib/Text/CSV_PP.pm
branches/upstream/libtext-csv-perl/current/t/50_utf8.t
branches/upstream/libtext-csv-perl/current/t/70_rt.t
branches/upstream/libtext-csv-perl/current/t/75_hashref.t
branches/upstream/libtext-csv-perl/current/t/80_diag.t
Modified: branches/upstream/libtext-csv-perl/current/Changes
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/Changes?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/Changes (original)
+++ branches/upstream/libtext-csv-perl/current/Changes Wed Jun 18 17:23:59 2008
@@ -1,4 +1,10 @@
Revision history for Perl extension Text::CSV.
+
+1.06 Wed Jun 18 14:35:40 2008
+ - updated the compatibility for Text::CSV_XS version 0.51
+ * set binary => 1 when UTF8 flag marked.
+ * removed the max of 255 for bind_columns.
+ - made the value type of $csv->{_STRING} corresponding to XS.
1.05 Fri May 2 13:15:49 2008
- fixed quoting process in combine.
Modified: branches/upstream/libtext-csv-perl/current/MANIFEST
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/MANIFEST?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/MANIFEST (original)
+++ branches/upstream/libtext-csv-perl/current/MANIFEST Wed Jun 18 17:23:59 2008
@@ -26,6 +26,7 @@
t/76_magic.t array_ref from magig (useless for Text::CSV_PP)
t/80_diag.t Error diagnostics
+files/utf8.csv A UTF-8 encode test file
t/util.pl Extra test utilities
META.yml Module meta-data (added by MakeMaker)
Modified: branches/upstream/libtext-csv-perl/current/META.yml
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/META.yml?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/META.yml (original)
+++ branches/upstream/libtext-csv-perl/current/META.yml Wed Jun 18 17:23:59 2008
@@ -1,6 +1,6 @@
--- #YAML:1.0
name: Text-CSV
-version: 1.05
+version: 1.06
abstract: comma-separated values manipulator (using XS or PurePerl)
license: ~
author:
Modified: branches/upstream/libtext-csv-perl/current/README
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/README?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/README (original)
+++ branches/upstream/libtext-csv-perl/current/README Wed Jun 18 17:23:59 2008
@@ -1,4 +1,4 @@
-Text::CSV version 1.04
+Text::CSV version 1.06
========================
comma-separated values manipulator
Added: branches/upstream/libtext-csv-perl/current/files/utf8.csv
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/files/utf8.csv?rev=21848&op=file
==============================================================================
--- branches/upstream/libtext-csv-perl/current/files/utf8.csv (added)
+++ branches/upstream/libtext-csv-perl/current/files/utf8.csv Wed Jun 18 17:23:59 2008
@@ -1,0 +1,1 @@
+"Øl/Vin",0
Modified: branches/upstream/libtext-csv-perl/current/lib/Text/CSV.pm
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/lib/Text/CSV.pm?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/lib/Text/CSV.pm (original)
+++ branches/upstream/libtext-csv-perl/current/lib/Text/CSV.pm Wed Jun 18 17:23:59 2008
@@ -5,14 +5,14 @@
use Carp ();
BEGIN {
- $Text::CSV::VERSION = '1.05';
+ $Text::CSV::VERSION = '1.06';
$Text::CSV::DEBUG = 0;
}
# if use CSV_XS, requires version
my $Module_XS = 'Text::CSV_XS';
my $Module_PP = 'Text::CSV_PP';
-my $XS_Version = '0.43';
+my $XS_Version = '0.51';
my $Is_Dynamic = 0;
@@ -270,9 +270,9 @@
=head1 VERSION
- 1.05
-
-This module is compatible with Text::CSV_XS B<0.43> or later.
+ 1.06
+
+This module is compatible with Text::CSV_XS B<0.51> or later.
=head2 BINARY MODE
@@ -283,6 +283,15 @@
my $csv = Text::CSV->new({binary => 1});
See to L<Text::CSV_XS/Embedded newlines>.
+
+=head2 Unicode (UTF8)
+
+On parsing (both for C<getline ()> and C<parse ()>), if the source is
+marked being UTF8, then parsing that source will mark all fields that
+are marked binary will also be marked UTF8.
+
+On combining (C<print ()> and C<combine ()>), if any of the combining
+fields was marked UTF8, the resulting string will be marked UTF8.
=head1 SPECIFICATION
@@ -636,7 +645,7 @@
=head2 bind_columns
-Takes a list of references to scalars (max 255) to store the fields fetched
+Takes a list of references to scalars to store the fields fetched
C<getline ()> in. When you don't pass enough references to store the
fetched fields in, C<getline ()> will fail. If you pass more than there are
fields to return, the remaining references are left untouched.
@@ -800,7 +809,7 @@
$csv->SetDiag (0);
-Use to reset the diagnosticts if you are dealing with errors.
+Use to reset the diagnostics if you are dealing with errors.
=head2 Some methods are Text::CSV only.
Modified: branches/upstream/libtext-csv-perl/current/lib/Text/CSV_PP.pm
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/lib/Text/CSV_PP.pm?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/lib/Text/CSV_PP.pm (original)
+++ branches/upstream/libtext-csv-perl/current/lib/Text/CSV_PP.pm Wed Jun 18 17:23:59 2008
@@ -11,7 +11,7 @@
use vars qw($VERSION);
use Carp ();
-$VERSION = '1.13';
+$VERSION = '1.14';
sub PV { 0 }
sub IV { 1 }
@@ -53,7 +53,6 @@
3002 => "EHR - getline_hr () called before column_names ()",
3003 => "EHR - bind_columns () and column_names () fields count mismatch",
3004 => "EHR - bind_columns () only accepts refs to scalars",
- 3005 => "EHR - bind_columns () takes 254 refs max",
3006 => "EHR - bind_columns () did not pass enough refs for parsed fields",
3007 => "EHR - bind_columns needs refs to writeable scalars",
3008 => "EHR - unexpected error in bound fields",
@@ -95,7 +94,17 @@
BEGIN {
if ( $] < 5.006 ) {
- $INC{'bytes.pm'} = 1; # dummy
+ $INC{'bytes.pm'} = 1 unless $INC{'bytes.pm'}; # dummy
+ no strict 'refs';
+ *{"utf8::is_utf8"} = sub { 0; };
+ }
+ elsif ( $] < 5.008 ) {
+ no strict 'refs';
+ *{"utf8::is_utf8"} = sub { 0; };
+ }
+ elsif ( !defined &utf8::is_utf8 ) {
+ require Encode;
+ *utf8::is_utf8 = *Encode::is_utf8;
}
eval q| require Scalar::Util |;
@@ -186,14 +195,13 @@
}
return $context ? @diag : $diagobj;
-# return $context ? (0 + $diagobj, "$diagobj") : $diagobj;
}
################################################################################
# string
################################################################################
*string = \&_string;
sub _string {
- $_[0]->{_STRING};
+ defined $_[0]->{_STRING} ? ${ $_[0]->{_STRING} } : undef;
}
################################################################################
# fields
@@ -233,6 +241,9 @@
unless (defined $column) {
$column = '';
next;
+ }
+ elsif ( !$binary ) {
+ $binary = 1 if utf8::is_utf8 $column;
}
if (!$binary and $column =~ /[^\x09\x20-\x7E]/) {
@@ -261,7 +272,7 @@
}
}
- $self->{_STRING} = join($sep, @part) . $self->{eol};
+ $self->{_STRING} = \do { join($sep, @part) . $self->{eol} };
$self->{_STATUS} = 1;
return $self->{_STATUS};
@@ -327,6 +338,8 @@
my $pos = 0;
+ my $utf8 = 1 if utf8::is_utf8( $line ); # if UTF8 marked, flag on.
+
for my $col ($line =~ /$re_split/g) {
if ($keep_meta_info) {
@@ -336,7 +349,7 @@
$pos += length $col;
- if (!$binary and $col =~ /[^\x09\x20-\x7E]/) { # Binary character, binary off
+ if ( ( !$binary and !$utf8 ) and $col =~ /[^\x09\x20-\x7E]/) { # Binary character, binary off
if ( $col =~ $re_quoted ) {
$self->_set_error_diag(
$col =~ /\n([^\n]*)/ ? (2021, $pos - 1 - length $1)
@@ -352,6 +365,12 @@
: (2037, $pos - length $col) # Binary character in unquoted field, binary off
);
}
+ $palatable = 0;
+ last;
+ }
+
+ if ( ($utf8 and !$binary) and $col =~ /\n|\0/ ) { # \n still needs binary (Text::CSV_XS 0.51 compat)
+ $self->_set_error_diag(2021, $pos);
$palatable = 0;
last;
}
@@ -601,10 +620,6 @@
if ( $self->{_COLUMN_NAMES} && @refs != @{$self->{_COLUMN_NAMES}} ) {
$self->SetDiag( 3003 );
- }
-
- if ( @refs > 255 ) {
- $self->SetDiag( 3005 );
}
if ( grep { ref $_ ne "SCALAR" } @refs ) { # why don't use grep?
@@ -784,6 +799,14 @@
comma-separated values. As its name suggests, L<Text::CSV_XS>
is a XS module and Text::CSV_PP is a Puer Perl one.
+=head2 Unicode (UTF8)
+
+On parsing (both for C<getline ()> and C<parse ()>), if the source is
+marked being UTF8, then parsing that source will mark all fields that
+are marked binary will also be marked UTF8.
+
+On combining (C<print ()> and C<combine ()>), if any of the combining
+fields was marked UTF8, the resulting string will be marked UTF8.
=head1 FUNCTIONS
@@ -1289,7 +1312,7 @@
$csv->SetDiag (0);
-Use to reset the diagnosticts if you are dealing with errors.
+Use to reset the diagnostics if you are dealing with errors.
=head1 DIAGNOSTICS
@@ -1353,8 +1376,6 @@
=item 3004 "EHR - bind_columns () only accepts refs to scalars"
-=item 3005 "EHR - bind_columns () takes 254 refs max"
-
=item 3006 "EHR - bind_columns () did not pass enough refs for parsed fields"
=item 3007 "EHR - bind_columns needs refs to writeable scalars"
Modified: branches/upstream/libtext-csv-perl/current/t/50_utf8.t
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/t/50_utf8.t?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/t/50_utf8.t (original)
+++ branches/upstream/libtext-csv-perl/current/t/50_utf8.t Wed Jun 18 17:23:59 2008
@@ -7,21 +7,27 @@
BEGIN {
if ($] < 5.008) {
- plan skip_all => "UTF8 tests useless in this ancient perl version";
+ plan skip_all => "UTF8 tests useless in this ancient perl version";
+ }
+ else {
+ plan tests => 67;
+ }
}
- else {
- plan tests => 57;
- }
-}
BEGIN {
$ENV{PERL_TEXT_CSV} = 0;
require_ok "Text::CSV";
plan skip_all => "Cannot load Text::CSV" if $@;
+# require_ok "Text::CSV_XS";
+# plan skip_all => "Cannot load Text::CSV_XS" if $@;
require "t/util.pl";
-}
+ }
-my $csv = Text::CSV->new ({ binary => 1, always_quote => 1 });
+# No binary => 1, as UTF8 is supposed to be allowed without it
+my $csv = Text::CSV->new ({
+ always_quote => 1,
+ keep_meta_info => 1,
+ });
# Special characters to check:
# 0A = \n 2C = , 20 = 22 = "
@@ -41,7 +47,7 @@
# characters in 128..255
) {
my ($u, $msg) = @$test;
- utf8::encode ($u);
+ ($u = "$u\x{0123}") =~ s/.$//; # Make sure it's marked UTF8
my @in = ("", " ", $u, "");
my $exp = join ",", map { qq{"$_"} } @in;
@@ -58,3 +64,28 @@
is_binary ($in[$_], $out[$_], "field $_ $msg");
}
}
+
+# Test if the UTF8 part is accepted, but the \n is not
+is ($csv->parse (qq{"\x{0123}\n\x{20ac}"}), 0, "\\n still needs binary");
+is ($csv->binary, 0, "bin flag still unset");
+is ($csv->error_diag + 0, 2021, "Error 2021");
+
+# As all utf tests are skipped for older pers, It's safe to use 3-arg open this way
+my $file = "files/utf8.csv";
+SKIP: {
+ open my $fh, "<:encoding(utf8)", $file or
+ skip "Cannot open UTF-8 test file", 6;
+
+ my $row;
+ ok ($row = $csv->getline ($fh), "read/parse");
+
+ is ($csv->is_quoted (0), 1, "First field is quoted");
+ is ($csv->is_quoted (1), 0, "Second field is not quoted");
+ is ($csv->is_binary (0), 1, "First field is binary");
+ is ($csv->is_binary (1), 0, "Second field is not binary");
+
+ ok (utf8::valid ($row->[0]), "First field is valid utf8");
+
+ $csv->combine (@$row);
+ ok (utf8::valid ($csv->string), "Combined string is valid utf8");
+ }
Modified: branches/upstream/libtext-csv-perl/current/t/70_rt.t
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/t/70_rt.t?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/t/70_rt.t (original)
+++ branches/upstream/libtext-csv-perl/current/t/70_rt.t Wed Jun 18 17:23:59 2008
@@ -7,9 +7,10 @@
use Test::More tests => 69;
BEGIN {
+ $ENV{PERL_TEXT_CSV} = 0;
use_ok "Text::CSV", ();
plan skip_all => "Cannot load Text::CSV" if $@;
- }
+}
my $rt_no;
my %input;
Modified: branches/upstream/libtext-csv-perl/current/t/75_hashref.t
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/t/75_hashref.t?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/t/75_hashref.t (original)
+++ branches/upstream/libtext-csv-perl/current/t/75_hashref.t Wed Jun 18 17:23:59 2008
@@ -10,8 +10,10 @@
$ENV{PERL_TEXT_CSV} = 0;
use_ok "Text::CSV", ();
plan skip_all => "Cannot load Text::CSV" if $@;
+# use_ok "Text::CSV_XS", ();
+# plan skip_all => "Cannot load Text::CSV_XS" if $@;
}
-#print Text::CSV_XS->VERSION, "\n";
+
open FH, ">_test.csv";
print FH <<EOC;
code,name,price,description
@@ -29,7 +31,7 @@
eval { $csv->column_names (@$args) };
like ($@, qr/^EHR/, "croak");
is ($csv->error_diag () + 0, 3001, "Bad args to column_names");
-}
+ }
ok ($csv->column_names ("name"), "One single name");
is ($csv->column_names (undef), undef, "reset column_names");
@@ -68,8 +70,6 @@
eval { $csv->bind_columns ({}, {}, {}, {}) };
is ($csv->error_diag () + 0, 3004, "bad arg types");
is ($csv->column_names (undef), undef, "reset column_names");
-eval { $csv->bind_columns ((\$code) x 300) };
-is ($csv->error_diag () + 0, 3005, "too many args");
ok ($csv->bind_columns (\($code, $name, $price)), "Bind columns");
eval { $csv->column_names ("foo") };
@@ -97,9 +97,12 @@
($code, $name, $price, $desc, $foo) = (101 .. 105);
ok ($csv->getline (*FH), "fetch less than expected");
is_deeply ( [ $code, $name, $price, $desc, $foo ],
- [ 2, "Drinks", "82.78", "Drinks", 105 ], "unfetched not reset");
+ [ 2, "Drinks", "82.78", "Drinks", 105 ], "unfetched not reset");
-ok ($csv->bind_columns (\1, \2, \3, \""), "bind too many columns");
+my @foo = (0) x 0x012345;
+ok ($csv->bind_columns (\(@foo)), "bind a lot of columns");
+
+ok ($csv->bind_columns (\1, \2, \3, \""), "bind too constant columns");
is ($csv->getline (*FH), undef, "fetch to read-only ref");
is ($csv->error_diag () + 0, 3008, "Read-only");
Modified: branches/upstream/libtext-csv-perl/current/t/80_diag.t
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-perl/current/t/80_diag.t?rev=21848&op=diff
==============================================================================
--- branches/upstream/libtext-csv-perl/current/t/80_diag.t (original)
+++ branches/upstream/libtext-csv-perl/current/t/80_diag.t Wed Jun 18 17:23:59 2008
@@ -65,8 +65,10 @@
parse_err 2034, 2, qq{1, "bar",2};
parse_err 2037, 1, qq{\0 };
-diag ("Next line should be an error message");
-$csv->error_diag ();
+unless (($ENV{AUTOMATED_TESTING} || 0) == "1") {
+ diag ("Next line should be an error message");
+ $csv->error_diag ();
+}
is (Text::CSV->new ({ ecs_char => ":" }), undef, "Unsupported option");
More information about the Pkg-perl-cvs-commits
mailing list