mirror of
https://github.com/curl/curl.git
synced 2026-08-25 14:43:33 +03:00
badwords: move into ./scripts, speed up
- 'badwords' is now a target in Makefile.am - change badwords.txt to specify plain "words" instead of regexes so the script can build single regexes when scanning, which makes the script perform much faster (~6 times faster) Closes #20869
This commit is contained in:
Daniel Stenberg
parent
248dd9e55f
commit
713287188e
7 changed files with 123 additions and 78 deletions
|
|
@ -22,11 +22,12 @@
|
|||
#
|
||||
###########################################################################
|
||||
|
||||
EXTRA_DIST = coverage.sh completion.pl firefox-db2pem.sh checksrc.pl checksrc-all.pl \
|
||||
mk-ca-bundle.pl mk-unity.pl schemetable.c cd2nroff nroff2cd cdall cd2cd managen \
|
||||
dmaketgz maketgz release-tools.sh verify-release cmakelint.sh mdlinkcheck \
|
||||
CMakeLists.txt perlcheck.sh pythonlint.sh spacecheck.pl randdisable wcurl \
|
||||
top-complexity extract-unit-protos .checksrc
|
||||
EXTRA_DIST = coverage.sh completion.pl firefox-db2pem.sh checksrc.pl \
|
||||
checksrc-all.pl mk-ca-bundle.pl mk-unity.pl schemetable.c cd2nroff nroff2cd \
|
||||
cdall cd2cd managen dmaketgz maketgz release-tools.sh verify-release \
|
||||
cmakelint.sh mdlinkcheck CMakeLists.txt perlcheck.sh pythonlint.sh \
|
||||
spacecheck.pl randdisable wcurl top-complexity extract-unit-protos \
|
||||
.checksrc badwords badwords.ok badwords.txt
|
||||
|
||||
dist_bin_SCRIPTS = wcurl
|
||||
|
||||
|
|
|
|||
165
scripts/badwords
Executable file
165
scripts/badwords
Executable file
|
|
@ -0,0 +1,165 @@
|
|||
#!/usr/bin/env perl
|
||||
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
#
|
||||
# SPDX-License-Identifier: curl
|
||||
#
|
||||
# bad[:=]correct
|
||||
#
|
||||
# If separator is '=', the string will be compared case sensitively.
|
||||
# If separator is ':', the check is done case insensitively.
|
||||
#
|
||||
# To add white listed uses of bad words that are removed before checking for
|
||||
# the bad ones:
|
||||
#
|
||||
# ---(accepted word)
|
||||
#
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
my @whitelist = (
|
||||
# ignore what looks like URLs
|
||||
'(^|\W)((https|http|ftp):\/\/[a-z0-9\-._~%:\/?\#\[\]\@!\$&\'\(\)*+,;=]+)',
|
||||
# ignore bolded sections
|
||||
'\*\*(.*?)\*\*');
|
||||
my %alt;
|
||||
my %exactcase;
|
||||
my $skip_indented = 1;
|
||||
|
||||
if($ARGV[0] eq "-a") {
|
||||
shift @ARGV;
|
||||
$skip_indented = 0;
|
||||
}
|
||||
my %wl;
|
||||
if($ARGV[0] eq "-w") {
|
||||
shift @ARGV;
|
||||
my $file = shift @ARGV;
|
||||
open(W, "<$file") or die "Cannot open '$file': $!";
|
||||
while(<W>) {
|
||||
if(/^#/) {
|
||||
# allow #-comments
|
||||
next;
|
||||
}
|
||||
if(/^([^:]*):(\d*):(.*)/) {
|
||||
$wl{"$1:$2:$3"}=1;
|
||||
#print STDERR "whitelisted $1:$2:$3\n";
|
||||
}
|
||||
}
|
||||
close(W);
|
||||
}
|
||||
|
||||
my @w;
|
||||
my @exact;
|
||||
while(<STDIN>) {
|
||||
chomp;
|
||||
if($_ =~ /^#/) {
|
||||
next;
|
||||
}
|
||||
if($_ =~ /^---(.*)/) {
|
||||
push @whitelist, $1;
|
||||
}
|
||||
elsif($_ =~ /^(.*)([:=])(.*)/) {
|
||||
my ($bad, $sep, $better)=($1, $2, $3);
|
||||
$alt{$bad} = $better;
|
||||
if($sep eq "=") {
|
||||
push @exact, $bad;
|
||||
}
|
||||
else {
|
||||
push @w, $bad;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Build a single combined regex for case-insensitive words
|
||||
my $re_ci;
|
||||
if(@w) {
|
||||
my $pat = join('|', map { '\b'.quotemeta($_).'\b' } @w);
|
||||
$re_ci = qr/($pat)/i;
|
||||
}
|
||||
|
||||
# Build a single combined regex for case-sensitive (exact) words
|
||||
my $re_cs;
|
||||
if(@exact) {
|
||||
my $pat = join('|', map { '\b'.quotemeta($_).'\b' } @exact);
|
||||
$re_cs = qr/($pat)/;
|
||||
}
|
||||
|
||||
my $errors = 0;
|
||||
|
||||
sub highlight {
|
||||
my ($p, $w, $in, $f, $l) = @_;
|
||||
|
||||
my $c = length($p)+1;
|
||||
my $ch = "$f:$l:$w";
|
||||
if($wl{$ch}) {
|
||||
# whitelisted filename + line + word
|
||||
return;
|
||||
}
|
||||
$ch = $f . "::" . $w;
|
||||
if($wl{$ch}) {
|
||||
# whitelisted filename + word
|
||||
return;
|
||||
}
|
||||
|
||||
print STDERR "$f:$l:$c: error: found bad word \"$w\"\n";
|
||||
printf STDERR " %4d | %s\n", $l, $in;
|
||||
printf STDERR " | %*s^%s\n", length($p), " ",
|
||||
"~" x (length($w)-1);
|
||||
printf STDERR " maybe use \"%s\" instead?\n", $alt{$w};
|
||||
$errors++;
|
||||
}
|
||||
|
||||
sub file {
|
||||
my ($f) = @_;
|
||||
my $l = 0;
|
||||
open(F, "<$f");
|
||||
while(<F>) {
|
||||
my $in = $_;
|
||||
$l++;
|
||||
chomp $in;
|
||||
if($skip_indented && $in =~ /^ /) {
|
||||
next;
|
||||
}
|
||||
# remove the link part
|
||||
$in =~ s/(\[.*\])\(.*\)/$1/g;
|
||||
# remove backticked texts
|
||||
$in =~ s/\`.*\`//g;
|
||||
# remove whitelisted patterns (pre-compiled)
|
||||
for my $p (@whitelist) {
|
||||
$in =~ s/$p//g;
|
||||
}
|
||||
# case-insensitive bad words
|
||||
if($re_ci) {
|
||||
while($in =~ /^(.*)$re_ci/i) {
|
||||
highlight($1, $2, $in, $f, $l);
|
||||
last;
|
||||
}
|
||||
}
|
||||
# case-sensitive (exact) bad words
|
||||
if($re_cs) {
|
||||
while($in =~ /^(.*)$re_cs/) {
|
||||
highlight($1, $2, $in, $f, $l);
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
close(F);
|
||||
}
|
||||
|
||||
my @filemasks = @ARGV;
|
||||
open(my $git_ls_files, '-|', 'git', 'ls-files', '--', @filemasks) or die "Failed running git ls-files: $!";
|
||||
my @files;
|
||||
while(my $each = <$git_ls_files>) {
|
||||
chomp $each;
|
||||
push @files, $each;
|
||||
}
|
||||
close $git_ls_files;
|
||||
|
||||
my $onum = scalar(@files);
|
||||
my $num;
|
||||
for my $e (@files) {
|
||||
#printf STDERR "Complete: %d%%\r", $num++ * 100 / $onum;
|
||||
file($e);
|
||||
}
|
||||
|
||||
exit $errors;
|
||||
8
scripts/badwords.ok
Normal file
8
scripts/badwords.ok
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
#
|
||||
# SPDX-License-Identifier: curl
|
||||
#
|
||||
# whitelisted uses of bad words
|
||||
# file:[line]:rule
|
||||
docs/FAQ.md::will
|
||||
docs/FAQ.md::Will
|
||||
97
scripts/badwords.txt
Normal file
97
scripts/badwords.txt
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
#
|
||||
# SPDX-License-Identifier: curl
|
||||
#
|
||||
back-end:backend
|
||||
e-mail:email
|
||||
run-time:runtime
|
||||
set-up:setup
|
||||
tool chain:toolchain
|
||||
tool-chain:toolchain
|
||||
wild-card:wildcard
|
||||
wild card:wildcard
|
||||
thread safe:thread-safe
|
||||
thread unsafe:thread-unsafe
|
||||
multi thread:multi-thread
|
||||
it's:it is
|
||||
aren't:are not
|
||||
can't:cannot
|
||||
could've:could have
|
||||
couldn't:could not
|
||||
didn't:did not
|
||||
doesn't:does not
|
||||
don't:do not
|
||||
haven't:have not
|
||||
i'd:I would
|
||||
i'll:I will
|
||||
i'm:I am
|
||||
i've:I have
|
||||
isn't:is not
|
||||
it'd:it would
|
||||
it'll:it will
|
||||
might've:might have
|
||||
needn't:need not
|
||||
should've:should have
|
||||
shouldn't:should not
|
||||
that's:that is
|
||||
there's:there is
|
||||
they'd:They would
|
||||
they'll:They will
|
||||
they're:They are
|
||||
they've:They have
|
||||
this'll:this will
|
||||
wasn't:was not
|
||||
we'd:we would
|
||||
we'll:we will
|
||||
we're:we are
|
||||
we've:we have
|
||||
weren't:were not
|
||||
won't:will not
|
||||
would've:would have
|
||||
wouldn't:would not
|
||||
you'd:you would
|
||||
you'll:you will
|
||||
you're:you are
|
||||
you've:you have
|
||||
a html:an html
|
||||
a http:an http
|
||||
a ftp:an ftp
|
||||
a IPv4:an IPv4
|
||||
a IPv6:an IPv6
|
||||
url= URL
|
||||
internet=Internet
|
||||
isation:ization
|
||||
So=Rewrite it somehow?
|
||||
And=Rewrite it somehow?
|
||||
But=Rewrite it somehow?
|
||||
sub-directory:subdirectory
|
||||
web page:webpage
|
||||
host name:hostname
|
||||
host names:hostnames
|
||||
file name:filename
|
||||
file names:filenames
|
||||
fist:first
|
||||
user name:username
|
||||
user names:usernames
|
||||
pass phrase:passphrase
|
||||
will:rewrite to present tense
|
||||
32 bit:32-bit
|
||||
16 bit:16-bit
|
||||
64 bit:64-bit
|
||||
32-bits:32 bits
|
||||
16-bits:16 bits
|
||||
64-bits:64 bits
|
||||
very:rephrase using an alternative word
|
||||
just:rephrase using an alternative word
|
||||
Curl=curl
|
||||
cURL=curl
|
||||
Libcurl=libcurl
|
||||
LibCurl=libcurl
|
||||
---WWW::Curl
|
||||
---NET::Curl
|
||||
---Curl Corporation
|
||||
manpages:man pages
|
||||
manpage:man page
|
||||
favour:favor
|
||||
basically:rephrase?
|
||||
However,:rephrase?
|
||||
Loading…
Add table
Add a link
Reference in a new issue