summaryrefslogtreecommitdiff
path: root/notes.txt
blob: 0722c205c8300d3902c6f73c8fab3a186f65de26 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
/tmp/wireshark/configure --prefix=/tmp/wsroot --with-ssl --with-gtk2 --without-gtk3
# CMake rocks!
CC=clang CXX=clang++ \
cmake -DCMAKE_INSTALL_PREFIX=/tmp/wsroot -DENABLE_GTK3=0 -DENABLE_PORTAUDIO=0 -DENABLE_QT5=0 -DENABLE_GEOIP=0 -DENABLE_KERBEROS=0 -DENABLE_SMI=0
-DCMAKE_BUILD_TYPE=Debug /tmp/wireshark -DCMAKE_C_FLAGS=-fsanitize=address -DCMAKE_CXX_FLAGS=-fsanitize=address
-DCMAKE_EXPORT_COMPILE_COMMANDS=1
# Do not set CMAKE_C_FLAGS_DEBUG, it breaks (missing symbols)
#clang-check -p /tmp/wsbuild epan/dissectors/packet-whois.c -ast-dump

v1.99.0-rc1-578-gdbd409d clang cmake: 15s; make -j16: 1m34s
v1.99.0-rc1-588-gc9694dc gcc cmake: 13.6s; make -j16: 1m37s

# libgcrypt RC2 fixing (accepted in git
# http://lists.gnupg.org/pipermail/gcrypt-devel/2013-October/002428.html)
# Fix compile issue (missing fig2dev in doc, Makefile.in in tests is not
# generated (Makefile/autotools of the repo too old?))
sed '/^tests\/Makefile$/d' -i configure.ac &&
sed '/SUBDIRS/s/ doc tests//' -i Makefile.am && ./autogen.sh
mkdir build && cd build
../configure --disable-static --disable-padlock-support --prefix=/tmp/libgcrypt/prefix
make install
# for libgcrypt-config test in wireshark ./configure
PATH=/tmp/libgcrypt/prefix/bin:$PATH
# for cmake add:
-DGCRYPT_INCLUDE_DIR=/tmp/libgcrypt/prefix/include -DGCRYPT_LIBRARY=/tmp/libgcrypt/prefix/lib/libgcrypt.so

# Build cyassl for AES-CCM testing
patch -p1 < cyassl-Implement-SSLKEYLOGFILE-support-for-ClientRandom.patch
autoreconf -fiv
./configure --enable-{opensslextra,dtls,ipv6,sniffer,aes{c,g}cm,camellia,md2,dsa,ecc,psk,webserver,sni}
make
examples/server/server -p 4433
SSLKEYLOGFILE=premaster.txt examples/client/client -l AES256-SHA -p 4433
see also cyassl-test (in this repo) for testing all supported ciphers
# Show a list of cipher suite from ServerHello and the HTTP version (or the
# number of the ServerHello if decryption failed).
/tmp/wsbuild/tshark -r cyassl-tcp.pcapng.gz -ohttp.ssl.port:4430-4433 \
 -ossl.keylog_file:premaster.txt -ossl.psk:1a2b3c4d -Tfields \
 -e frame.number -e ssl.handshake.ciphersuite -e http.request.version \
 -Y 'ssl.handshake.type==2||ssl.record.content_type==23' |
 awk '$2~/0x/{if(n)print n;printf("%s ",$2);n=$1}$2=="HTTP/1.0"{print $2;n=""}END{if(n)print n}'

# create suites.txt from http://www.iana.org/assignments/tls-parameters/tls-parameters-4.csv
gawk -n -F '[,"]+' '$4~/^TLS/{print int($2)*0x100+int($3), $4}' tls-parameters-4.csv > suites.txt
# find which suites are not supported yet (unsupported.txt)
gawk -n -vsrc=/tmp/wireshark/epan/dissectors/packet-ssl-utils.c -F'[ {,]+' 'BEGIN{while(getline <src)if(/^ *\{.*,KEX_/)a[int($2)]=1}{if(!a[int($1)])print}' suites.txt
# find which ciphers openssl supports
openssl ciphers -tls1 -V ALL:NULL | tr , \  | while read x y _ name _;do echo $((x*0x100+y)) $name;done | sort -n > openssl-supported-ciphers.txt
# find which ciphers are not yet supported (unsupported-new is from above)
grep -E "$(cut -d' ' -f1 unsuppported-new.txt  openssl-supported-ciphers.txt | sort | uniq -d | tr '\n' '|' | sed 's/|$//')" unsuppported-new.txt -w


# Get keys from NSS applications:
SSLKEYLOGFILE=premaster.txt firefox

# command to use key file from NSS
/tmp/wsroot/bin/wireshark -o ssl.keylog_file:$PWD/s_client-keys.txt s_client.capng -o http.ssl.port:4433 -o ssl.debug_file:s_client-debug.txt

# Command to look for relation between cipher and mode (stream vs cbc)
grep epan/dissectors/packet-ssl-utils.c -e '^ *{.*,KEX' | column -s, -t | sort -k 4,4 -k 11,11

# pipe openssl
stdbuf -oL openssl s_server -CAfile server.crt -cert server.crt -key server.pem -www -cipher ALL 2>&1 | awk '/ACCEPT/{print (++n) " " $0}!/ACCEPT/{print}{fflush()}'
openssl ciphers | tr : '\n' > ciphers.txt
# trigger tests:
openssl ciphers|tr : '\n' | while read i;do echo;echo $i;curl --cacert /tmp/snif/pki/server.crt https://localhost:4433 -o /dev/null --ciphers $i -v;done
# "fail" file is above output
awk 'BEGIN{while(getline<"fail"){if(/ACCEPT/){n=$1}else if(/error/){fails[n]=1}}} {if(!fails[NR])print}' ciphers.txt


# convert CipherSuite from RFC to code
xsel | ./generate-wireshark-cs | sed s/{/,/ | sort -t, -n -k2,2 | sed s/,/{/
# check for differences between existing ciphers and new ones from X clipboard
ssort(){ sed s/{/,/ | sort -n -t, -k2,2 | sed s/,/{/; }
grep ,KEX_ packet-ssl-utils.c | ssort > 1;(cat 1; xsel) | sort -k1,1 -t} -u | ssort > 2; colordiff -u 1 2

# command to sort by Au(th) and show official TLS names
openssl ciphers -V | sort -k6,6 |
gawk '{split($1,a,",");$1=strtonum(a[1])*256+strtonum(a[2])}{print}' |
sort -k6,6 | ./number-to-name.awk -vcmd='cat suites.txt' | column -t | less


# dump CLIENT_RANDOM for every cipher
openssl ciphers|tr : '\n' | grep -vE '^(PSK|SRP|ECDHE-ECDSA|ECDH)-|-DSS-' | while read cipher; do (echo 'GET / HTTP/1.0';sleep .1) | openssl s_client -connect localhost:4433 -cipher $cipher -msg 2>&1 | awk '/Master-Key:/{key=$2} {b=1;e=16;if(l==3)b=7;if(l==1)e=6;for(i=b;i<=e;i++)s=s$i;if(l--==1)r[s]=1}/ ClientHello|ServerHello$/{l=3;s=""} END{for(rnd in r)print "CLIENT_RANDOM",rnd,key}';done > all/s_client-keys.txt
# dump CLIENT_RANDOM for every cipher for *.local.al.lekensteyn.nl with TLS disabled
for url in $(grep -E '/(IDEA-CBC-SHA|EXP-RC2-CBC-MD5)\.' -i ssl3/ok.txt); do host="${url##*/}"; (printf "GET / HTTP/1.1\r\nHost: $host\r\n\r\n";sleep .2) | openssl s_client -connect "$host" -CApath /etc/nginx/certs -no_tls1 -msg 2>&1 | awk '/Master-Key:/{key=$2} {b=1;e=16;if(l==3)b=7;if(l==1)e=6;for(i=b;i<=e;i++)s=s$i;if(l--==1)r[s]=1}/ ClientHello|ServerHello$/{l=3;s=""} END{for(rnd in r)print "CLIENT_RANDOM",rnd,key}'; done >> /tmp/snif/ssl3/premaster.txt

# fetch a list of hosts to visit
</etc/nginx/sites/ciphertest.conf awk -F '[:; ]+' '/listen/{port=$4}/server_name/&&$5{print "https://"$5":"port}' > urls.txt
# Get good and bad cipher suites wrt web server certs
rm ok.txt nok.txt;time while read url; do curl -ks "$url" -o /dev/null && echo $url >> ok.txt || echo $url >> nok.txt;done < urls.txt
# same as above, but restrict to OpenSSL ciphers during request
rm ok.txt nok.txt;time while read url; do cipher="${url%%.*}";cipher="${cipher##*/}";curl -ks "$url" -o /dev/null --ciphers "${cipher^^}" && echo $url >> ok.txt || echo $url >> nok.txt;done < urls.txt
# same test, but using openssl instead of curl
for url in $(cat res/ok.txt); do host="${url##*/}"; echo;echo;echo _____ $host;(printf "GET / HTTP/1.1\r\nHost: $host\r\n\r\n";sleep .2) | openssl s_client -connect "$host" -CApath /etc/nginx/certs; done 2>&1 | tee s_client-all-res-ok.txt


The following OpenSSL cipher suites do not connect to nginx (obsolete):
EXP-EDH-DSS-DES-CBC-SHA
EXP-EDH-RSA-DES-CBC-SHA
PSK-RC4-SHA
PSK-3DES-EDE-CBC-SHA
PSK-AES128-CBC-SHA
PSK-AES256-CBC-SHA
ECDH-RSA-RC4-SHA
ECDH-RSA-DES-CBC3-SHA
ECDH-RSA-AES128-SHA
ECDH-RSA-AES256-SHA
SRP-RSA-3DES-EDE-CBC-SHA
SRP-DSS-3DES-EDE-CBC-SHA
SRP-RSA-AES-128-CBC-SHA
SRP-DSS-AES-128-CBC-SHA
SRP-RSA-AES-256-CBC-SHA
SRP-DSS-AES-256-CBC-SHA
ECDH-RSA-AES128-SHA256
ECDH-RSA-AES256-SHA384
ECDH-RSA-AES128-GCM-SHA256
ECDH-RSA-AES256-GCM-SHA384

Groupable to:
- EXP-EDH-{DSS,RSA}
- PSK
- ECDH-RSA (obsolete, it is supported with correct cert)
- SRP

Not supported by GnuTLS (source:
http://backreference.org/2009/11/18/openssl-vs-gnutls-cipher-names/)
- TLS-SRP (Secure Remote Password)
- PSK (Pre-Shared Key)
- ECDH-{RSA,ECDSA} (not ECDHE-RSA) (source: wikipedia)

Missing cipher suite support (from
http://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4):
- SRP
- KRB5
- PSK
- ARIA
- mode CCM

libgcrypt RC2 (40-bit keys) implementation is named "RFC2268_40". This does not
seem to work with TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 though. I think that the
unimplemented RFC2268_128 algo should be used. As mentioned in RFC 2246 (TLS
1.0), section 6.3.1. Export key generation example, this RC2 cipher suite has a
final_client_write_key length of 128 bits.

# Generate RSA stuff
openssl genrsa -out server.pem
openssl req -new -x509 -key server.pem -out server.crt -days 3650 -subj "/CN=*.local.al.lekensteyn.nl"
# Generate dsa params, privkey and signed pubkey
openssl dsaparam 1024 -out dsaparam.pem
openssl gendsa dsaparam.pem -out dsa.pem
openssl req -new -key dsa.pem -x509 -days 3650 -out dsa.crt -subj "/CN=*.local.al.lekensteyn.nl"
# Generete EC params (secp112r1 cert does not work, "no shared cipher" error)
# secp256r1 is supported by chromium (and secp{384,521}r1 too)
openssl ecparam -name prime192v1 -out ec.pem -genkey
openssl req -new -key ec.pem -x509 -days 3650 -out ec.crt -subj "/CN=*.local.al.lekensteyn.nl/OU=EC"

# Test session resumption
tshark -i veth0 -p -w ssl-session-id.pcapng -P -f tcp
openssl s_server -www -key server.key -msg -cipher DHE-RSA-AES128-SHA256
curl -kvv --ciphers DHE-RSA-AES128-SHA256 https://10.9.0.1/{,}

http://wiki.wireshark.org/SampleCaptures?action=AttachFile
// Find all extensions
$('a[href*="do=get"]').toArray().map(function (e){ var m=e.href.match(/\.([^.?]+(\.gz|bz2)?)$/); return m?m[1]:"";}).reduce(function (prev, cur){ if(prev.indexOf(cur)==-1)prev.push(cur);return prev;}, [])
// Find all extensions, count per extension
$('a[href*="do=get"]').toArray().map(function (e){ var m=e.href.match(/\.([^.?]+(\.gz|bz2)?)$/); return m?m[1]:"";}).reduce(function (prev, cur){ if(cur in prev)prev[cur]++;else prev[cur]=1;return prev;},{})
# Download captures
wget -e robots=off -nc -r -l 1 --accept-regex='.*do=get.*(p?cap|pcapng)(\.gz)?$' --ignore-case http://wiki.wireshark.org/SampleCaptures?action=AttachFile
# Symlink files (retains original name for easier updates)
mkdir captures && cd captures && ln -s ../wiki.wireshark.org .;
find wiki.wireshark.org/ -name '*target=*' | php -r 'while ($line = fgets(STDIN)) { $line = trim($line); symlink($line, urldecode(preg_replace("#.*target=#", "", $line))); }'
# Find duplicates, list by md5, mtime, size, filename
md5sum * | sort | uniq -w32 -D | while read sum file; do echo $sum $(date +"%Y-%m-%d %H:%M" -r "$file") "$(du -hD "$file")"; done
# Archive captures/ (537M uncompressed, 223M gz, 177M xz)
time tar cJhf captures.tar.xz --owner=root --group=root --exclude=wiki.wireshark.org captures/

# Save capture files from remote
save(){ local d="pc:/tmp/wireshark-caps"; rsync -vasL "$d/captures/$1" "$d/tshark-${2:-1}/$1."{out,err} ./"${1%.*}/";}
# Retrieve 2724 attachments (509MB) from https://www.wireshark.org/~darkjames/capture-files.txt
time cut -d: -f2 ../capture-files.txt | sed 's,^,https:,' | xargs wget -a ../capture-files-wget.txt -nv --content-disposition
# Rename the captured files such that attachment id is included
time grep id= ../capture-files-wget.txt | sed 's/.*attachment.cgi?id=\([0-9]*\) \[[0-9/]*\] -> "/\1 /;s/" \[[12]\]$//' | while read id name; do mv "$name" -vi "$id-${name%.[0-9]}" ;done
# Rename name that wget could not handle
mv '4993-=?UTF-8?Q?PPP=2D=E8=BD=AC=E6=8D=A2=E5=90=8E=E6=96=87=E4=BB=B6=2D=E6=89=93=E5=BC=80=E6=95=B0=E6=8D=AE=E5=8C=85=E6=97=A0=E6=98=BE=E7=A4=BA=2Ecap?=' '4993-PPP-转换后文件-打开 数据包无显示.cap'

# logcat testing
for file in logcat-v*.txt;do for type in logcat logcat-brief logcat-process logcat-tag logcat-time logcat-thread logcat-threadtime logcat-long;do /tmp/wsbuild/run/tshark -r "$file" -F $type -w out/$file-$type.txt;done;done

ASN.1 dissectors (make the "export" file first):
asn1/x509sat$ ../../tools/asn2wrs.py -E -b -r Syntax -p x509sat -c ./x509sat.cnf -D . SelectedAttributeTypes.asn
asn1/x509if$ ../../tools/asn2wrs.py -b -p x509if -c ./x509if.cnf -s ./packet-x509if-template -D . -O ../../epan/dissectors InformationFramework.asn ServiceAdministration.asn
This solves this error:
:0: UserWarning: Missing tag information for imported type DirectoryString from SelectedAttributeTypes (SelectedAttributeTypes)

# Find all headers mentioning "fmt" or "fmt" pointers, but without attribute
grep --exclude=\*.c -Hnr -e '\* *\(fmt\|format\) *\([,)]\|$\)' -C2 --color=always | awk '{s=s $0"\n"}/^[0-9m\[K\x1b]*--/{ if(!att){print s} att=0;s="" } /GNUC_PRINTF/{ att=1 }'

Compile benchmarks on i7-3770 (clean builddir, sources in tmpfs) using
v1.99.3rc0-233-gbccb7f7 (3 trials)
                total           cmake           make/ninja      make/ninja(2nd)
Makefile -j8    154.6s+/-.084s  21.86s+/-.082s  132.7s+/-.039s  2.01s+/-.021s
Makefile -j10   153.1s+/-.163s  21.81s+/-.198s  131.3s+/-.323s  2.07s+/-.096s
Makefile -j16   153.9s+/-.366s  21.89s+/-.102s  132.0s+/-.334s  2.02s+/-.011s
Ninja (-j10)    141.0s+/-.261s  19.90s+/-.214s  121.0s+/-.073s  .366s+/-.012s

# CCC_CC and CCC_CXX prevent fallback to gcc.
export CCC_CC=clang CCC_CXX=clang++
cmake -GNinja -DCMAKE_C_COMPILER=/usr/lib/clang-analyzer/scan-build/ccc-analyzer -DCMAKE_CXX_COMPILER=/usr/lib/clang-analyzer/scan-build/c++-analyzer /tmp/wireshark/ -DCMAKE_BUILD_TYPE=Debug -DDISABLE_WERROR=1 -DENABLE_EXTRA_COMPILER_WARNINGS=1 -DCMAKE_EXPORT_COMPILE_COMMANDS=1
script -c 'time scan-build -o $PWD/scanresults ninja' build.log

Enable MPX (GCC 6)
-mmpx -fcheck-pointer-bounds
-fno-sanitize=address -fno-sanitize=undefined -fuse-ld=bfd
# Have to disable --as-needed and -pie in CMakeLists.txt too, otherwise
# -Wl,--as-needed results in:
/usr/lib/gcc/x86_64-pc-linux-gnu/6.1.1/../../../../lib/libmpxwrappers.so: undefined reference to `get_bd'
# alternatively, add -lmpx -lmpxwrappers to linker flags, see
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77267
# -pie gives:
/usr/bin/ld.bfd: run/libgtkui.a(expert_comp_dlg.c.o)(.text+0x8526): reloc against `expert_group_vals': error 4
/usr/bin/ld.bfd: final link failed: Nonrepresentable section on output
# fixed in binutils 2.28 https://sourceware.org/bugzilla/show_bug.cgi?id=20550


# Find all hf_ variables
set pagination off
set logging on hf.txt
info variables ^hf_
# Then find all hf_ int fields that are not registered
awk <hf-v2.3.0rc0-820-g8b04755.txt '/^File/{sub(":",""); file=$NF} /^(static )?g?int hf_/&&!/\[|\*/{sub(";",""); print "if " $NF "<1"; print " printf \"%s %s %d\\n\", \""file"\", \""$NF"\", "$NF; print "end"}' > commands
set pagination off
set logging on vals.txt
source commands

# Test ciphers with GnuTLS (CCM)
# -CURVE-ALL:-SIGN-ALL to slim down Hello handshake size.
psktool -p test.psk -u test
gnutls-serv --http -p 4433 --pskpasswd=test.psk --noticket
 --priority=+AEAD:+PSK:+AES-128-CCM:+AES-256-CCM-8:+AES-128-GCM
# client (wait a bit in order to generate multiple packets)
for s in 'GET / HTTP/1.1' 'Host: localhost' ''; do sleep .5; echo "$s"; done |
gnutls-cli -p 443 localhost --pskusername=test --pskkey=...
 --priority=%NO_SESSION_HASH:%NO_TICKETS:+MAC-ALL:+PSK:+AES-128-CCM:-CURVE-ALL:-SIGN-ALL
# For DTLS, just append "-u", server will echo request as response
(for s in 'DTLS1.2 test using GnuTLS 3.5.8' 'Works for me!'; do sleep .5; echo "$s";done)