perl去除重复内容的脚本代码(重复行+数组重复字段)

# Date:2011-09-28
# Version :1.0
use warnings;
use strict;

my %hash;
my $script = $0; # Get the script name

sub usage
{
        printf("Usage:n");
        printf("perl $script <source_file> <dest_file>n");

}

# If the number of parameters less than 2 ,exit the script
if ( $#ARGV+1 < 2) {

        &usage;
        exit 0;
}

my $source_file = $ARGV[0]; #File need to remove duplicate rows
my $dest_file = $ARGV[1]; # File after remove duplicates rows

open (FILE,"<$source_file") or die "Cannot open file $!n";
open (SORTED,">$dest_file") or die "Cannot open file $!n";

while(defined (my $line = <FILE>))
{
        chomp($line);
        $hash{$line} += 1;
        # print "$line,$hash{$line}n";
}

foreach my $k (keys %hash) {
        print SORTED "$k,$hash{$k}n";#改行打印出列和该列出现的次数到目标文件
}
close (FILE);
close (SORTED);

代码三：

通过perl脚本，删除数据组中重复的字段

#!/usr/bin/perl
use strict;
my %hash;
my @array = (1..10,5,20,2,3,4,5,5);
#grep 保存符合条件的元素
@array = grep { ++$hash{$_} < 2 } @array;
print join(" ",@array);
print "n";