#!usr/bin/env python
with open("all.txt") as input_:
    with open("out.txt", "a") as out:
        for x in set(input_.readlines()):
            out.write(x)

#!/usr/bin/perl -w # Удаление повторяющихся строк 
# by [dei] 
############ Config ############# 
$file = 'log.txt';  
#############################  
open(F1,"< $file");  
@lines = <F1>; 
close F1;  
for($i=0;$i<@lines;$i++) {        
      for($i2=$i+1;$i2<@lines;$i2++) {         
            f($lines[$i] eq $lines[$i2]) { splice @lines, $i2, 1;}     
    } 
}  
open(F2,"> $file"); 
print F2 join('',@lines); 
close F2;

sort -u file.txt

grep -vF "`sort f1.txt | uniq -d`" f1.txt