[sr-dev] [kamailio/kamailio] Segfault start Kamailio 5.5 (#2736)

[ReznikovAlexei]

### Description

I am upgraded Kamailio from 5.4.5 to 5.5, but i have got segfault when I try start Kamailio with custom config, with default config Kamailio start work done.

#### Reproduction

/usr/sbin/kamailio -Ee -d -DD -P /var/run/kamailio/kamailio.pid -m 64 -M 32 -f /etc/kazoo/kamailio/kamailio.cfg -w /run/kamailio/

#### Debugging Data
```
[***@hostname ~]# gdb /usr/sbin/kamailio /run/kamailio/core.29437
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/sbin/kamailio...Reading symbols from /usr/lib/debug/usr/sbin/kamailio.debug...done.
done.
[New LWP 29437]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/usr/sbin/kamailio -Ee -d -DD -P /var/run/kamailio/kamailio.pid -m 64 -M 32 -f'.
Program terminated with signal 11, Segmentation fault.
#0 0x00007f31b5397d26 in __memcpy_ssse3_back () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.6-13.el7.x86_64 cyrus-sasl-lib-2.1.26-23.el7.x86_64 elfutils-libelf-0.176-5.el7.x86_64 elfutils-libs-0.176-5.el7.x86_64 glibc-2.17-323.el7_9.x86_64 jansson-2.10-1.el7.x86_64 json-c-0.11-4.el7_0.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-50.el7.x86_64 libattr-2.4.46-13.el7.x86_64 libcap-2.22-11.el7.x86_64 libcom_err-1.42.9-19.el7.x86_64 libcurl-7.29.0-59.el7_9.1.x86_64 libevent-2.0.21-4.el7.x86_64 libgcc-4.8.5-44.el7.x86_64 libidn-1.28-4.el7.x86_64 librabbitmq-0.8.0-3.el7.x86_64 libselinux-2.5-15.el7.x86_64 libssh2-1.8.0-4.el7.x86_64 libstdc++-4.8.5-44.el7.x86_64 libuuid-2.23.2-65.el7_9.1.x86_64 libxml2-2.9.1-6.el7.5.x86_64 mariadb-libs-5.5.68-1.el7.x86_64 nspr-4.25.0-2.el7_9.x86_64 nss-3.53.1-3.el7_9.x86_64 nss-softokn-freebl-3.53.1-6.el7_9.x86_64 nss-util-3.53.1-1.el7_9.x86_64 openldap-2.4.44-22.el7.x86_64 openssl-libs-1.0.2k-21.el7_9.x86_64 pcre-8.32-17.el7.x86_64 systemd-libs-219-78.el7_9.3.x86_64 xz-libs-5.2.2-1.el7.x86_64 zlib-1.2.7-19.el7_9.x86_64
(gdb)
(gdb)
(gdb)
(gdb) bt full
#0 0x00007f31b5397d26 in __memcpy_ssse3_back () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f31b2dcdfed in pv_parse_hdr_name (sp=0x7f31b38b27d0, in=0x7ffc78f0e1f0) at pv_core.c:3272
s = {s = 0x0, len = 29437}
p = 0x0
nsp = 0x0
hdr = {type = 2029052412, name = {s = 0x7ffc78f0e200 "", len = -1282654080}, body = {s = 0x7ffc78f0e20c "\001", len = -1282725936}, len = 2029052384, parsed = 0x1006ccf31, next = 0x49500000001}
__FUNCTION__ = "pv_parse_hdr_name"
#2 0x000000000062ee65 in pv_parse_spec2 (in=0x7f31b38b27b8, e=0x7f31b38b27d0, silent=0) at core/pvapi.c:969
p = 0x7f31b38b283b ")[0])"
s = {s = 0x7f31b38b2836 "X-CID)[0])", len = 5}
pvname = {s = 0x7f31b38b2832 "hdr(X-CID)[0])", len = 3}
pvstate = 2
tr = 0x0
pte = 0x7f31b32d3c30
n = 0
__FUNCTION__ = "pv_parse_spec2"
#3 0x000000000062a71e in pv_cache_add (name=0x7ffc78f0e440) at core/pvapi.c:359
pvn = 0x7f31b38b27b8
pvid = 949637875
p = 0xffffffff000072fd <Address 0xffffffff000072fd out of bounds>
__FUNCTION__ = "pv_cache_add"
#4 0x000000000062bf56 in pv_spec_lookup (name=0x7ffc78f0e520, len=0x7ffc78f0e51c) at core/pvapi.c:498
pvs = 0x0
tname = {s = 0x7f31b38c9db8 "$(hdr(X-CID)[0])", len = 16}
__FUNCTION__ = "pv_spec_lookup"
#5 0x000000000063241d in pv_parse_format (in=0x7ffc78f0e660, el=0x7f31b38b26a0) at core/pvapi.c:1194
p = 0x7f31b38c9db8 "$(hdr(X-CID)[0])"
p0 = 0x280007e2a15 <Address 0x280007e2a15 out of bounds>
n = 1
e = 0x7f31b38b2730
e0 = 0x0
s = {s = 0x7f31b38c9db8 "$(hdr(X-CID)[0])", len = 16}
len = 16
__FUNCTION__ = "pv_parse_format"
#6 0x000000000059faa1 in fix_param (type=256, param=0x7f31b38c9bf8) at core/sr_module.c:1214
p = 0x7f31b38b2690
name = {s = 0x7f31b38c9db8 "$(hdr(X-CID)[0])", len = 16}
s = {s = 0x20 <Address 0x20 out of bounds>, len = -1282726872}
num = 0
err = 32764
__FUNCTION__ = "fix_param"
#7 0x00000000005a00d8 in fix_param_types (types=256, param=0x7f31b38c9bf8) at core/sr_module.c:1336
ret = 2029056944
t = 256
#8 0x0000000000657f66 in fixup_spve_null (param=0x7f31b38c9bf8, param_no=1) at core/mod_fix.c:564
ret = 0
fp = 0x0
__FUNCTION__ = "fixup_spve_null"
#9 0x00007f31aa22bfac in fixup_hvalue_param (param=0x7f31b38c9bf8, param_no=2) at textopsx.c:622
No locals.
#10 0x00007f31aa22cd49 in fixup_hname_str (param=0x7f31b38c9bf8, param_no=2) at textopsx.c:719
---Type <return> to continue, or q <return> to quit---
No locals.
#11 0x00007f31aa2345ef in append_hf_value_fixup (param=0x7f31b38c9bf8, param_no=2) at textopsx.c:1644
res = 0
__FUNCTION__ = "append_hf_value_fixup"
#12 0x00000000006743c9 in fix_actions (a=0x7f31b38c9b80) at core/route.c:932
t = 0x7f31b38c9b80
p = 0x7f31b380c668
tmp = 0x7f31b38c5ae8 ""
tmp_p = 0x7f31b38c9db8
ret = 0
i = 1
cmd = 0x7f31b32fb050
s = {s = 0x7ffc78f0eb70 "\360\355\360x\374\177", len = 7141887}
he = 0x41c380 <_start>
ip = {af = 3012334440, len = 32561, u = {addrl = {139851442459504, 8589934592}, addr32 = {3012334448, 32561, 0, 2}, addr16 = {37744, 45964, 32561, 0, 0, 0, 2, 0},
addr = "p\223\214\263\061\177\000\000\000\000\000\000\002\000\000"}}
si = 0x7ffc78f0eb70
lval = 0x7f31b38c6418
rve = 0x41c380 <_start>
err_rve = 0x642e38 <sr_event_exec+415>
rve_type = 32764
err_type = 2029054464
expected_type = 32764
rv = 0x4651dc <fix_rval_expr+783>
rve_param_no = 0
__FUNCTION__ = "fix_actions"
#13 0x000000000066fd5b in fix_actions (a=0x7f31b38c9e38) at core/route.c:723
t = 0x7f31b38c9e38
p = 0x7ffc78f0ee40
tmp = 0xb5605380 <Address 0xb5605380 out of bounds>
tmp_p = 0x7f31b38c6040
ret = 0
i = 2
cmd = 0x7f31b32fe998
s = {s = 0x7f31b3895f40 "LIS_REPLY", len = 9}
he = 0x7ffc78f0ed10
ip = {af = 2029055136, len = 32764, u = {addrl = {7947659, 140722337541280}, addr32 = {7947659, 0, 2029055136, 32764}, addr16 = {17803, 121, 0, 0, 60576, 30960, 32764, 0},
addr = "\213Ey\000\000\000\000\000\240\354\360x\374\177\000"}}
si = 0x100000400
lval = 0x7f31b3894c78
rve = 0x7f31b38c9368
err_rve = 0x0
rve_type = RV_INT
err_type = RV_NONE
expected_type = RV_NONE
rv = 0x7f31b3896938
rve_param_no = 0
__FUNCTION__ = "fix_actions"
#14 0x0000000000680d97 in fix_rl (rt=0xb9e5a0 <main_rt>) at core/route.c:2102
i = 104
ret = 0
#15 0x0000000000680dce in fix_rls () at core/route.c:2118
---Type <return> to continue, or q <return> to quit---
ret = 0
#16 0x0000000000436704 in main (argc=14, argv=0x7ffc78f0f3b8) at main.c:3047
cfg_stream = 0x2a7d040
c = -1
r = 0
tmp = 0x7ffc78f0f850 ""
tmp_len = 1472
port = 960
proto = 32561
ahost = 0x0
aport = 0
options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 539315467
rfd = 4
debug_save = 1
debug_flag = 1
dont_fork_cnt = 2
n_lst = 0x7ffc78f0f270
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600},
st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, {
name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {
name = 0x7df623 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {
name = 0x7df641 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {
name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
(gdb)
(gdb)
(gdb) info locals
cfg_stream = 0x2a7d040
c = -1
r = 0
tmp = 0x7ffc78f0f850 ""
tmp_len = 1472
port = 960
proto = 32561
ahost = 0x0
aport = 0
options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 539315467
rfd = 4
debug_save = 1
debug_flag = 1
dont_fork_cnt = 2
n_lst = 0x7ffc78f0f270
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600},
st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, {
name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id",
has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1,
flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {
name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
(gdb)
cfg_stream = 0x2a7d040
c = -1
r = 0
tmp = 0x7ffc78f0f850 ""
tmp_len = 1472
port = 960
proto = 32561
ahost = 0x0
aport = 0
options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 539315467
rfd = 4
debug_save = 1
debug_flag = 1
dont_fork_cnt = 2
n_lst = 0x7ffc78f0f270
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600},
st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, {
name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id",
has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1,
flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {
name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
(gdb)
cfg_stream = 0x2a7d040
c = -1
r = 0
tmp = 0x7ffc78f0f850 ""
tmp_len = 1472
port = 960
proto = 32561
ahost = 0x0
aport = 0
options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 539315467
rfd = 4
debug_save = 1
debug_flag = 1
dont_fork_cnt = 2
n_lst = 0x7ffc78f0f270
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600},
st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, {
name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id",
has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1,
flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {
name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
(gdb)
cfg_stream = 0x2a7d040
c = -1
r = 0
tmp = 0x7ffc78f0f850 ""
tmp_len = 1472
port = 960
proto = 32561
ahost = 0x0
aport = 0
options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 539315467
rfd = 4
debug_save = 1
debug_flag = 1
dont_fork_cnt = 2
n_lst = 0x7ffc78f0f270
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600},
st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, {
name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id",
has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1,
flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {
name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
(gdb)
cfg_stream = 0x2a7d040
c = -1
r = 0
tmp = 0x7ffc78f0f850 ""
tmp_len = 1472
port = 960
proto = 32561
ahost = 0x0
aport = 0
options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 539315467
rfd = 4
debug_save = 1
debug_flag = 1
dont_fork_cnt = 2
n_lst = 0x7ffc78f0f270
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600},
st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, {
name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id",
has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1,
flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {
name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
(gdb)
cfg_stream = 0x2a7d040
c = -1
r = 0
tmp = 0x7ffc78f0f850 ""
tmp_len = 1472
port = 960
proto = 32561
ahost = 0x0
aport = 0
options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 539315467
rfd = 4
debug_save = 1
debug_flag = 1
dont_fork_cnt = 2
n_lst = 0x7ffc78f0f270
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600},
st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, {
name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id",
has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1,
flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {
name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
(gdb) list
1981 int main(int argc, char** argv)
1982 {
1983
1984 FILE* cfg_stream;
1985 int c,r;
1986 char *tmp;
1987 int tmp_len;
1988 int port;
1989 int proto;
1990 char *ahost = NULL;
(gdb)
```

#### Log Messages


```
[***@hostname ~]# /usr/sbin/kamailio -Ee -d -DD -P /var/run/kamailio/kamailio.pid -m 64 -M 32 -f /etc/kazoo/kamailio/kamailio.cfg -w /run/kamailio/
0(30249) INFO: tls [tls_init.c:503]: init_tls_compression(): disabling compression...
0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "kzE"
0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [k/107] in [$(kzE{kz.json,From}{uri.user})] at [6 (4)]
0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter
0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "kzE"
0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [k/107] in [$(kzE{kz.json,Realm})] at [6 (4)]
0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter
0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "kzE"
0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [k/107] in [$(kzE{kz.json,Realm})] at [6 (4)]
0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter
0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "kzE"
0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [k/107] in [$(kzE{kz.json,Realm})] at [6 (4)]
0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter
0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "subs"
0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [t/116] in [$subs(to_user)] at [6 (5)]
0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter
0(30249) INFO: pv [pv_shv.c:60]: shvar_init_locks(): locks array size 16
0(30249) INFO: mqueue [mqueue_mod.c:257]: mq_param(): mqueue param: [presence_last_notity|0]
0(30249) INFO: mqueue [mqueue_mod.c:257]: mq_param(): mqueue param: [node_track|0]
0(30249) INFO: mqueue [mqueue_mod.c:257]: mq_param(): mqueue param: [node_heartbeat|0]
Listening on
udp: myip1:5060
tcp: myip1:5060
Aliases:
udp: myip1:5060
tcp: myip1:5060

0(30249) WARNING: <core> [core/daemonize.c:348]: daemonize(): pid file contains old pid, replacing pid
0(30249) NOTICE: nat_traversal [nat_traversal.c:1845]: mod_init(): keeping alive dialogs is disabled because the dialog module is not loaded
0(30249) NOTICE: regex [regex_mod.c:168]: mod_init(): 'file' parameter is not set, group matching disabled
0(30249) WARNING: tls [tls_init.c:796]: tls_h_mod_init_f(): openssl bug #1491 (crash/mem leaks on low memory) workaround enabled (on low memory tls operations will fail preemptively) with free memory thresholds 1024 and 1024 bytes
Segmentation fault (core dumped)
[***@hostname ~]#

```

### Additional Information


```
[***@hostname ~]# kamailio -v
version: kamailio 5.5.0 (x86_64/linux) d4c1a1
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: d4c1a1
compiled on 07:50:25 May 6 2021 with gcc 4.8.5
```

```
[***@hostname ~]# rpm -qa | grep kamailio
kamailio-presence-5.5.0-0.el7.centos.x86_64
kamailio-mysql-5.5.0-0.el7.centos.x86_64
kamailio-websocket-5.5.0-0.el7.centos.x86_64
kamailio-http_async_client-5.5.0-0.el7.centos.x86_64
kamailio-xmpp-5.5.0-0.el7.centos.x86_64
kamailio-utils-5.5.0-0.el7.centos.x86_64
kamailio-uuid-5.5.0-0.el7.centos.x86_64
kamailio-kazoo-5.5.0-0.el7.centos.x86_64
kamailio-http_client-5.5.0-0.el7.centos.x86_64
kamailio-xmlops-5.5.0-0.el7.centos.x86_64
kamailio-outbound-5.5.0-0.el7.centos.x86_64
kamailio-debuginfo-5.5.0-0.el7.centos.x86_64
kamailio-5.5.0-0.el7.centos.x86_64
kamailio-tls-5.5.0-0.el7.centos.x86_64
kamailio-regex-5.5.0-0.el7.centos.x86_64
kamailio-jansson-5.5.0-0.el7.centos.x86_64

```
* **Operating System**:

<!--
Details about the operating system, the type: Linux (e.g.,: Debian 8.4, Ubuntu 16.04, CentOS 7.1, ...), MacOS, xBSD, Solaris, ...;
Kernel details (output of `uname -a`)
-->

```
[***@hostname ~]# cat /etc/redhat-release
CentOS Linux release 7.9.2009 (Core)

```

[Daniel-Constantin Mierla]

Seems to be related to the fixup of `append_hf_value()` function -- can you paste here all the lines in your config with this function?

[ReznikovAlexei]

My configuration many lines func "append_hf_value". I am comment all lines for this function, but i still get segfault.
I created new TT #2738 for this issue with new debug core and additional info about my configuration.

[Daniel-Constantin Mierla]

In the #2738, the backtrace indicated a crash due to fixup of xlog functions while processing the parameter:

```
"$ci|end|sent subscription $hdr(Subscription-State)\n"
```

I just tested with the next line in a kamailio.cfg used with kamailio 5.5:

```
xlog("$ci|end|sent subscription $hdr(Subscription-State)\n");
```

and it starts ok. So the problem is somewhere else, potentially a buffer overflow in completely different part of the code.

Can you list all loaded modules in your configuration (the list of `loadmodule` lines)?

[ReznikovAlexei]

my list modules
```
accounting-role.cfg:loadmodule "acc.so"
antiflood-role.cfg:loadmodule "pike.so"
db_kazoo.cfg:loadmodule "db_kazoo.so"
db_mysql.cfg:loadmodule "db_mysql.so"
db_postgres.cfg:loadmodule "db_postgres.so"
default.cfg:loadmodule "mqueue.so"
default.cfg:loadmodule "outbound.so"
default.cfg:loadmodule "stun.so"
default.cfg:loadmodule "path.so"
default.cfg:loadmodule "ctl.so"
default.cfg:loadmodule "cfg_rpc.so"
default.cfg:loadmodule "cfgutils.so"
default.cfg:loadmodule "corex.so"
default.cfg:loadmodule "uuid.so"
default.cfg:loadmodule "kex.so"
default.cfg:loadmodule "tm.so"
default.cfg:loadmodule "tmx.so"
default.cfg:loadmodule "sl.so"
default.cfg:loadmodule "rr.so"
default.cfg:loadmodule "maxfwd.so"
default.cfg:loadmodule "siputils.so"
default.cfg:loadmodule "textopsx.so"
default.cfg:loadmodule "sdpops.so"
default.cfg:loadmodule "htable.so"
default.cfg:loadmodule "rtimer.so"
default.cfg:loadmodule "evrexec.so"
default.cfg:loadmodule "xlog.so"
default.cfg:loadmodule "uac.so"
default.cfg:loadmodule "avp.so"
default.cfg:loadmodule "avpops.so"
default.cfg:loadmodule "uac_redirect.so"
default.cfg:loadmodule "jsonrpcs.so"
default.cfg:loadmodule "sqlops.so"
default.cfg:loadmodule "debugger.so"
default.cfg:loadmodule "statistics.so"
default.cfg:loadmodule "permissions.so"
dispatcher-role-5.1.cfg:loadmodule "dispatcher.so"
dispatcher-role-5.2.cfg:loadmodule "dispatcher.so"
dispatcher-role-5.4.cfg:loadmodule "dispatcher.so"
dispatcher-role-5.5.cfg:loadmodule "dispatcher.so"
e911-role.cfg:loadmodule "regex.so"
kamailio.cfg:loadmodule "ipops.so"
kamailio.cfg:loadmodule "pv.so"
kamailio.cfg:loadmodule "textops.so"
kazoo-bindings.cfg:loadmodule "kazoo.so"
lis-role.cfg:loadmodule "jansson.so"
lis-role.cfg:loadmodule "http_async_client.so"
lis-role.cfg:loadmodule "http_client.so"
lis-role.cfg:loadmodule "xmlops.so"
msrp-proxy.cfg:loadmodule "msrp.so"
nat-traversal-role.cfg:loadmodule "nathelper.so"
presence-role.cfg:loadmodule "nat_traversal.so"
presence-role.cfg:loadmodule "presence.so"
presence-role.cfg:loadmodule "presence_dialoginfo.so"
presence-role.cfg:loadmodule "presence_mwi.so"
presence-role.cfg:loadmodule "presence_xml.so"
registrar-role.cfg:loadmodule "auth.so"
registrar-role.cfg:loadmodule "usrloc.so"
registrar-role.cfg:loadmodule "registrar.so"
registrar-role.cfg:loadmodule "nathelper.so"
sanity.cfg:loadmodule "sanity.so"
sip_trace-role.cfg:loadmodule "siptrace.so"
tls-role.cfg:loadmodule "tls.so"
websockets-role.cfg:loadmodule "nathelper.so"
websockets-role.cfg:loadmodule "xhttp.so"
websockets-role.cfg:loadmodule "websocket.so"
```

[sergey-safarov]

Hi Daniel @miconda
Alexei can find a commit where the issue is introduced.
Should we make `git bisect`?

[Daniel-Constantin Mierla]

`loadmodule "db_kazoo.so"` - this is not a module offered by Kamailio project, we cannot help when private extensions are used.

[ReznikovAlexei]

@miconda This module is not used, he is not installed in system.
Kamailio is default and installed from official repo.
```
[***@hostname kamailio]# ls -la /usr/lib64/kamailio/modules/ | grep db_
-rwxr-xr-x. 1 root root 192264 May 6 07:55 db_cluster.so
-rwxr-xr-x. 1 root root 99168 May 6 07:55 db_flatstore.so
-rwxr-xr-x. 1 root root 227840 May 6 07:55 db_mysql.so
-rwxr-xr-x. 1 root root 247136 May 6 07:55 db_text.so
```
```
kamailio -v
version: kamailio 5.5.0 (x86_64/linux) d4c1a1
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: d4c1a1
compiled on 07:50:25 May 6 2021 with gcc 4.8.5
```

[Daniel-Constantin Mierla]

Then try to isolate why happens by disabling loading modules one by one (and commenting the modparams and functions for those modules).The crash happens at startup, so should be easy to go this way till kamailio starts ok. It should be one of the modules not used in the default config files, which likely has a buffer overflow.

[sergey-safarov]

Minimal config
```
loadmodule "ipops.so"
loadmodule "pv.so"
loadmodule "textops.so"
loadmodule "textopsx.so"

pv_buffer_slots = 30

#!substdef "!MAJOR!$(version(num){re.subst,/^(([^\.])*\.([^\.])*)\..*/\1/})!g"
#!substdef "!MY_HOSTNAME!$HN(f)!g"
#!substdef "!MY_WEBSOCKET_DOMAIN!$HN(d)!g"
#!substdef "!KAMAILIO_DBMS!$def(KZ_DB_MODULE)!g"

#!substdef "!MY_IP_ADDRESS!$HN(i)!g"
#!substdef "!SANITY_SUBST_CACHE_PERIOD!$def(SANITY_CACHE_PERIOD)!g"

#!substdef "!KZQ_CHECK_MEDIA_SERVER_INSERT!insert into dispatcher (setid, destination) select \$var(SetId), \"\$var(MediaUrl)\" from DUAL where not exists(select * from dispatcher where destination = \"\$var(MediaUrl)\")!g"
#!substdef "!KZQ_COUNT_SUBSCRIBERS!select event, (select count(*) from active_watchers b where presentity_uri = \"\$var(presentity)\" and b.event = a.event) count from event_list a!g"
#!substdef "!KZQ_HANDLE_NEW_SUBSCRIBE_DELETE1!delete from active_watchers where callid = \"\$ci\"!g"
#!substdef "!KZQ_HANDLE_NEW_SUBSCRIBE_DELETE2!delete from active_watchers where watcher_username=\"\$fU\" and presentity_uri=\"\$var(presentity_uri)\" and to_user=\"\$tU\" and watcher_domain=\"\$fd\" and event=\"\$hdr(Event)\"!g"
#!substdef "!KZQ_RESET_PUBLISHER_UPDATE!update active_watchers set expires = \$TS where id in (select * from (select b.id from presentity a inner join active_watchers b on a.username = b.to_user and a.domain = b.to_domain and a.event = b.event where a.sender = \"\$var(MediaUrl)\") AS presentity_temp)!g"
#!substdef "!KZQ_PRESENCE_SEARCH_DETAIL!select * from active_watchers_log where presentity_uri = \"\$var(presentity_uri)\"!g"
#!substdef "!KZQ_PRESENCE_SEARCH_SUMMARY!select * from active_watchers where watcher_domain = \"\$var(Domain)\"!g"
#!substdef "!KZQ_PRESENCE_RESET!delete from presentity where sender = \"\$var(MediaUrl)\"!g"

listen=tcp:127.0.0.1:5090

####### Routing Logic ########
route
{
$avp(device_id) = $hdr(X-Device-Id);
$avp(account_db) = $hdr(X-Account-Db);
$var(text) = $hdr(Contact);
$var(expires) = $hdr(Expires);
$var(header) = $hdr(X-KAZOO-Respond-With);
$var(xxxx) = $hdr(To);
$var(rr_base) = $hdr(Record-Route);
$xavp(hf=>X-AUTH-IP) = $hdr(X-AUTH-IP);
$xavp(hf=>X-AUTH-PORT) = $hdr(X-AUTH-PORT);
$var(LocalRoute) = $hdr(X-TM-Local);
$ru = $hdr(X-URN-Service);
append_hf_value("Call-Info", "$(hdr(X-NenaCallId)[0])");

if ($hdr(X-KAZOO-INVITE-FORMAT) == "route") {
$var(referred_by) = $hdr(Referred-By);
}

if ($hdr(X-Redirect-Server) != $null) {
$avp(destination_uri) = $hdr(X-KAZOO-AOR);
}

}
```

If `pv_buffer_slots` has value 18-24, 26, 27, 29, 30 then core created.
Tested 0e51ce1075f206a4441333f72c69fcc56f8d6855

Also important, `-m` and `-M` Kamailio arguments affect how long Kamailio trying to start.
But no matter how much memory provided, Kamailio will fail.

[sergey-safarov]

Issue added by commit 004190b2ebe62681ae1f4f65f18de1a9e430742d
Trying revert on master.

[sergey-safarov]

Issue fixed after reverting 004190b2ebe62681ae1f4f65f18de1a9e430742d

[Daniel-Constantin Mierla]

Thanks for digging in further and narrowing it down. I will analyze to see what is the reason, like the references to the buffers become invalid as the needs for defines/substdefs can exceed/overwrite them. It is interesting that it didn't surface for defines which had this kind of capability/behaviour for long time before, the commit referenced above extended for substdef to make it coherent, because this one makes a define behind.

[Daniel-Constantin Mierla]

Can you try with master branch or the patch from the commit referenced above?

[sergey-safarov]

Tested current master (c146ef490e1d7d35add7d3ee593f6d3d20e327ad).
Issue resolved.

[sergey-safarov]

Closed #2736.