-------------------------------------------------------------------------------------------- corpora: BNC, verb: halt, number of patterns: 3, sample size: 50, annotators(4): AV EK JT SC -------------------------------------------------------------------------------------------- +-----------------------+ | Annotators: AV vs. EK | +-----------------------+ | 1 1.a 1.c 1.s 2 2.a 2.s 3 u x | # prob ------+---------------------------------------------------+------------ 1 | 38 - - - - 1 - - - - | 39 0.78 1.a | - - - - - - - - - - | 0 0.00 1.c | 2 1 - - - - - - - - | 3 0.06 1.s | - - - - - 1 - - - - | 1 0.02 2 | - - - - 3 - - - - - | 3 0.06 2.a | - - - - - - - - - - | 0 0.00 2.s | - - - - - - - 1 - - | 1 0.02 3 | - - - - - - - 1 - - | 1 0.02 u | - - - - 1 - - - - - | 1 0.02 x | - - - - - - - - - 1 | 1 0.02 ------+---------------------------------------------------+------------ # | 40 1 0 0 4 2 0 2 0 1 | 50 prob | 0.80 0.02 0.00 0.00 0.08 0.04 0.00 0.04 0.00 0.02 | 1 sample size: 50 number of agreements: 43 ( 0.86) number of disagreements: 7 ( 0.14) Cohen's kappa = 0.622 +-----------------------+ | Annotators: AV vs. JT | +-----------------------+ | 1 1.a 1.c 1.s 2 2.a 2.s 3 3.a u x | # prob ------+--------------------------------------------------------+------------ 1 | 32 1 2 - - 1 - 2 1 - - | 39 0.78 1.a | - - - - - - - - - - - | 0 0.00 1.c | 2 1 - - - - - - - - - | 3 0.06 1.s | - - - - 1 - - - - - - | 1 0.02 2 | - - - - 3 - - - - - - | 3 0.06 2.a | - - - - - - - - - - - | 0 0.00 2.s | - - - - - - - 1 - - - | 1 0.02 3 | - - - - - - - 1 - - - | 1 0.02 3.a | - - - - - - - - - - - | 0 0.00 u | 1 - - - - - - - - - - | 1 0.02 x | - - - - - - - - - - 1 | 1 0.02 ------+--------------------------------------------------------+------------ # | 35 2 2 0 4 1 0 4 1 0 1 | 50 prob | 0.70 0.04 0.04 0.00 0.08 0.02 0.00 0.08 0.02 0.00 0.02 | 1 sample size: 50 number of agreements: 37 ( 0.74) number of disagreements: 13 ( 0.26) Cohen's kappa = 0.415 +-----------------------+ | Annotators: AV vs. SC | +-----------------------+ | 1 1.c 1.s 2 2.s 3 u x | # prob ------+-----------------------------------------+------------ 1 | 35 2 1 - - - 1 - | 39 0.78 1.c | 2 1 - - - - - - | 3 0.06 1.s | - - 1 - - - - - | 1 0.02 2 | - - - 3 - - - - | 3 0.06 2.s | - - - - - 1 - - | 1 0.02 3 | - - - - - 1 - - | 1 0.02 u | - - - - - - 1 - | 1 0.02 x | - - - - - - - 1 | 1 0.02 ------+-----------------------------------------+------------ # | 37 3 2 3 0 2 2 1 | 50 prob | 0.74 0.06 0.04 0.06 0.00 0.04 0.04 0.02 | 1.00 sample size: 50 number of agreements: 43 ( 0.86) number of disagreements: 7 ( 0.14) Cohen's kappa = 0.661 +-----------------------+ | Annotators: EK vs. JT | +-----------------------+ | 1 1.a 1.c 2 2.a 3 3.a x | # prob ------+-----------------------------------------+------------ 1 | 33 2 2 - - 2 1 - | 40 0.80 1.a | 1 - - - - - - - | 1 0.02 1.c | - - - - - - - - | 0 0.00 2 | 1 - - 3 - - - - | 4 0.08 2.a | - - - 1 1 - - - | 2 0.04 3 | - - - - - 2 - - | 2 0.04 3.a | - - - - - - - - | 0 0.00 x | - - - - - - - 1 | 1 0.02 ------+-----------------------------------------+------------ # | 35 2 2 4 1 4 1 1 | 50 prob | 0.70 0.04 0.04 0.08 0.02 0.08 0.02 0.02 | 1.00 sample size: 50 number of agreements: 40 ( 0.80) number of disagreements: 10 ( 0.20) Cohen's kappa = 0.533 +-----------------------+ | Annotators: EK vs. SC | +-----------------------+ | 1 1.a 1.c 1.s 2 2.a 3 u x | # prob ------+----------------------------------------------+------------ 1 | 37 - 2 - - - - 1 - | 40 0.80 1.a | - - 1 - - - - - - | 1 0.02 1.c | - - - - - - - - - | 0 0.00 1.s | - - - - - - - - - | 0 0.00 2 | - - - - 3 - - 1 - | 4 0.08 2.a | - - - 2 - - - - - | 2 0.04 3 | - - - - - - 2 - - | 2 0.04 u | - - - - - - - - - | 0 0.00 x | - - - - - - - - 1 | 1 0.02 ------+----------------------------------------------+------------ # | 37 0 3 2 3 0 2 2 1 | 50 prob | 0.74 0.00 0.06 0.04 0.06 0.00 0.04 0.04 0.02 | 1 sample size: 50 number of agreements: 43 ( 0.86) number of disagreements: 7 ( 0.14) Cohen's kappa = 0.651 +-----------------------+ | Annotators: JT vs. SC | +-----------------------+ | 1 1.a 1.c 1.s 2 2.a 3 3.a u x | # prob ------+---------------------------------------------------+------------ 1 | 30 - 3 - - - - - 2 - | 35 0.70 1.a | 2 - - - - - - - - - | 2 0.04 1.c | 2 - - - - - - - - - | 2 0.04 1.s | - - - - - - - - - - | 0 0.00 2 | - - - 1 3 - - - - - | 4 0.08 2.a | - - - 1 - - - - - - | 1 0.02 3 | 2 - - - - - 2 - - - | 4 0.08 3.a | 1 - - - - - - - - - | 1 0.02 u | - - - - - - - - - - | 0 0.00 x | - - - - - - - - - 1 | 1 0.02 ------+---------------------------------------------------+------------ # | 37 0 3 2 3 0 2 0 2 1 | 50 prob | 0.74 0.00 0.06 0.04 0.06 0.00 0.04 0.00 0.04 0.02 | 1 sample size: 50 number of agreements: 36 ( 0.72) number of disagreements: 14 ( 0.28) Cohen's kappa = 0.406 -------------------------------------------------------------------------------------------- Fleiss' kappa = 0.540 Overview of inter-annotator agreement: ----------------------------------------------------------- double verb size #N #A Cohens agrs % AV_EK halt 50 3 4 0.622 43 0.860 AV_JT halt 50 3 4 0.415 37 0.740 AV_SC halt 50 3 4 0.661 43 0.860 EK_JT halt 50 3 4 0.533 40 0.800 EK_SC halt 50 3 4 0.651 43 0.860 JT_SC halt 50 3 4 0.406 36 0.720 ----------------------------------------------------------- halt AV EK JT SC AV - 0.622 0.415 0.661 EK - - 0.533 0.651 JT - - - 0.406 SC - - - -