CINXE.COM

<!DOCTYPE html> <html lang="en-GB" class="no-js"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width"> <link rel="profile" href="https://gmpg.org/xfn/11"> <link rel="pingback" href="https://www.generalist.org.uk/blog/xmlrpc.php">  <script>(function(html){html.className = html.className.replace(/\bno-js\b/,'js')})(document.documentElement);</script> <title>data – Andrew Gray</title> <style type="text/css"> /** * GeSHi Dynamically Generated Stylesheet * -------------------------------------- * Dynamically generated stylesheet for bash * CSS class: , CSS id: * GeSHi (C) 2004 - 2007 Nigel McNie, 2007 - 2014 Benny Baumann * (http://qbnz.com/highlighter/ and http://geshi.org/) * -------------------------------------- */ .bash .de1, .bash .de2 {} .bash {font-family:monospace;} .bash .imp {font-weight: bold; color: red;} .bash li, .bash .li1 {font-weight: normal; vertical-align:top;} .bash .ln {width:1px;text-align:right;margin:0;padding:0 2px;vertical-align:top;} .bash .li2 {font-weight: bold; vertical-align:top;} .bash .kw1 {color: #000000; font-weight: bold;} .bash .kw2 {color: #c20cb9; font-weight: bold;} .bash .kw3 {color: #7a0874; font-weight: bold;} .bash .co0 {color: #666666; font-style: italic;} .bash .co1 {color: #800000;} .bash .co2 {color: #cc0000; font-style: italic;} .bash .co3 {color: #000000; font-weight: bold;} .bash .co4 {color: #666666;} .bash .es1 {color: #000099; font-weight: bold;} .bash .es2 {color: #007800;} .bash .es3 {color: #007800;} .bash .es4 {color: #007800;} .bash .es5 {color: #780078;} .bash .es_h {color: #000099; font-weight: bold;} .bash .br0 {color: #7a0874; font-weight: bold;} .bash .sy0 {color: #000000; font-weight: bold;} .bash .st0 {color: #ff0000;} .bash .st_h {color: #ff0000;} .bash .nu0 {color: #000000;} .bash .re0 {color: #007800;} .bash .re1 {color: #007800;} .bash .re2 {color: #007800;} .bash .re4 {color: #007800;} .bash .re5 {color: #660033;} .bash .ln-xtra, .bash li.ln-xtra, .bash div.ln-xtra {background-color: #ffc;} .bash span.xtra { display:block; } </style> <meta name='robots' content='max-image-preview:large' /> <link rel="alternate" type="application/rss+xml" title="Andrew Gray » Feed" href="https://www.generalist.org.uk/blog/feed/" /> <link rel="alternate" type="application/rss+xml" title="Andrew Gray » Comments Feed" href="https://www.generalist.org.uk/blog/comments/feed/" /> <link rel="alternate" type="application/rss+xml" title="Andrew Gray » data Tag Feed" href="https://www.generalist.org.uk/blog/tags/data/feed/" /> <script> window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/14.0.0\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/14.0.0\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/www.generalist.org.uk\/blog\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.4.5"}}; /*! This file is auto-generated */ !function(i,n){var o,s,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),r=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===r[t]})}function u(e,t,n){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\uddfa\ud83c\uddf3","\ud83c\uddfa\u200b\ud83c\uddf3")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!n(e,"\ud83e\udef1\ud83c\udffb\u200d\ud83e\udef2\ud83c\udfff","\ud83e\udef1\ud83c\udffb\u200b\ud83e\udef2\ud83c\udfff")}return!1}function f(e,t,n){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):i.createElement("canvas"),a=r.getContext("2d",{willReadFrequently:!0}),o=(a.textBaseline="top",a.font="600 32px Arial",{});return e.forEach(function(e){o[e]=t(a,e,n)}),o}function t(e){var t=i.createElement("script");t.src=e,t.defer=!0,i.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",s=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){i.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(s),u.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"}),a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=function(e){c(n=e.data),a.terminate(),t(n)})}catch(e){}c(n=f(s,u,p))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings); </script> <style id='wp-emoji-styles-inline-css'> img.wp-smiley, img.emoji { display: inline !important; border: none !important; box-shadow: none !important; height: 1em !important; width: 1em !important; margin: 0 0.07em !important; vertical-align: -0.1em !important; background: none !important; padding: 0 !important; } </style> <link rel='stylesheet' id='wp-block-library-css' href='https://www.generalist.org.uk/blog/wp-includes/css/dist/block-library/style.min.css?ver=6.4.5' media='all' /> <style id='wp-block-library-theme-inline-css'> .wp-block-audio figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-audio figcaption{color:hsla(0,0%,100%,.65)}.wp-block-audio{margin:0 0 1em}.wp-block-code{border:1px solid #ccc;border-radius:4px;font-family:Menlo,Consolas,monaco,monospace;padding:.8em 1em}.wp-block-embed figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-embed figcaption{color:hsla(0,0%,100%,.65)}.wp-block-embed{margin:0 0 1em}.blocks-gallery-caption{color:#555;font-size:13px;text-align:center}.is-dark-theme .blocks-gallery-caption{color:hsla(0,0%,100%,.65)}.wp-block-image figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-image figcaption{color:hsla(0,0%,100%,.65)}.wp-block-image{margin:0 0 1em}.wp-block-pullquote{border-bottom:4px solid;border-top:4px solid;color:currentColor;margin-bottom:1.75em}.wp-block-pullquote cite,.wp-block-pullquote footer,.wp-block-pullquote__citation{color:currentColor;font-size:.8125em;font-style:normal;text-transform:uppercase}.wp-block-quote{border-left:.25em solid;margin:0 0 1.75em;padding-left:1em}.wp-block-quote cite,.wp-block-quote footer{color:currentColor;font-size:.8125em;font-style:normal;position:relative}.wp-block-quote.has-text-align-right{border-left:none;border-right:.25em solid;padding-left:0;padding-right:1em}.wp-block-quote.has-text-align-center{border:none;padding-left:0}.wp-block-quote.is-large,.wp-block-quote.is-style-large,.wp-block-quote.is-style-plain{border:none}.wp-block-search .wp-block-search__label{font-weight:700}.wp-block-search__button{border:1px solid #ccc;padding:.375em .625em}:where(.wp-block-group.has-background){padding:1.25em 2.375em}.wp-block-separator.has-css-opacity{opacity:.4}.wp-block-separator{border:none;border-bottom:2px solid;margin-left:auto;margin-right:auto}.wp-block-separator.has-alpha-channel-opacity{opacity:1}.wp-block-separator:not(.is-style-wide):not(.is-style-dots){width:100px}.wp-block-separator.has-background:not(.is-style-dots){border-bottom:none;height:1px}.wp-block-separator.has-background:not(.is-style-wide):not(.is-style-dots){height:2px}.wp-block-table{margin:0 0 1em}.wp-block-table td,.wp-block-table th{word-break:normal}.wp-block-table figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-table figcaption{color:hsla(0,0%,100%,.65)}.wp-block-video figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-video figcaption{color:hsla(0,0%,100%,.65)}.wp-block-video{margin:0 0 1em}.wp-block-template-part.has-background{margin-bottom:0;margin-top:0;padding:1.25em 2.375em} </style> <style id='classic-theme-styles-inline-css'> /*! This file is auto-generated */ .wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none} </style> <style id='global-styles-inline-css'> body{--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--dark-gray: #111;--wp--preset--color--light-gray: #f1f1f1;--wp--preset--color--yellow: #f4ca16;--wp--preset--color--dark-brown: #352712;--wp--preset--color--medium-pink: #e53b51;--wp--preset--color--light-pink: #ffe5d1;--wp--preset--color--dark-purple: #2e2256;--wp--preset--color--purple: #674970;--wp--preset--color--blue-gray: #22313f;--wp--preset--color--bright-blue: #55c3dc;--wp--preset--color--light-blue: #e9f2f9;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--gradient--dark-gray-gradient-gradient: linear-gradient(90deg, rgba(17,17,17,1) 0%, rgba(42,42,42,1) 100%);--wp--preset--gradient--light-gray-gradient: linear-gradient(90deg, rgba(241,241,241,1) 0%, rgba(215,215,215,1) 100%);--wp--preset--gradient--white-gradient: linear-gradient(90deg, rgba(255,255,255,1) 0%, rgba(230,230,230,1) 100%);--wp--preset--gradient--yellow-gradient: linear-gradient(90deg, rgba(244,202,22,1) 0%, rgba(205,168,10,1) 100%);--wp--preset--gradient--dark-brown-gradient: linear-gradient(90deg, rgba(53,39,18,1) 0%, rgba(91,67,31,1) 100%);--wp--preset--gradient--medium-pink-gradient: linear-gradient(90deg, rgba(229,59,81,1) 0%, rgba(209,28,51,1) 100%);--wp--preset--gradient--light-pink-gradient: linear-gradient(90deg, rgba(255,229,209,1) 0%, rgba(255,200,158,1) 100%);--wp--preset--gradient--dark-purple-gradient: linear-gradient(90deg, rgba(46,34,86,1) 0%, rgba(66,48,123,1) 100%);--wp--preset--gradient--purple-gradient: linear-gradient(90deg, rgba(103,73,112,1) 0%, rgba(131,93,143,1) 100%);--wp--preset--gradient--blue-gray-gradient: linear-gradient(90deg, rgba(34,49,63,1) 0%, rgba(52,75,96,1) 100%);--wp--preset--gradient--bright-blue-gradient: linear-gradient(90deg, rgba(85,195,220,1) 0%, rgba(43,180,211,1) 100%);--wp--preset--gradient--light-blue-gradient: linear-gradient(90deg, rgba(233,242,249,1) 0%, rgba(193,218,238,1) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flow > .alignleft{float: left;margin-inline-start: 0;margin-inline-end: 2em;}body .is-layout-flow > .alignright{float: right;margin-inline-start: 2em;margin-inline-end: 0;}body .is-layout-flow > .aligncenter{margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > .alignleft{float: left;margin-inline-start: 0;margin-inline-end: 2em;}body .is-layout-constrained > .alignright{float: right;margin-inline-start: 2em;margin-inline-end: 0;}body .is-layout-constrained > .aligncenter{margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > :where(:not(.alignleft):not(.alignright):not(.alignfull)){max-width: var(--wp--style--global--content-size);margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > .alignwide{max-width: var(--wp--style--global--wide-size);}body .is-layout-flex{display: flex;}body .is-layout-flex{flex-wrap: wrap;align-items: center;}body .is-layout-flex > *{margin: 0;}body .is-layout-grid{display: grid;}body .is-layout-grid > *{margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;} .wp-block-navigation a:where(:not(.wp-element-button)){color: inherit;} :where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;} :where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;} .wp-block-pullquote{font-size: 1.5em;line-height: 1.6;} </style> <link rel='stylesheet' id='twentyfifteen-fonts-css' href='https://www.generalist.org.uk/blog/wp-content/themes/twentyfifteen/assets/fonts/noto-sans-plus-noto-serif-plus-inconsolata.css?ver=20230328' media='all' /> <link rel='stylesheet' id='genericons-css' href='https://www.generalist.org.uk/blog/wp-content/themes/twentyfifteen/genericons/genericons.css?ver=20201026' media='all' /> <link rel='stylesheet' id='twentyfifteen-style-css' href='https://www.generalist.org.uk/blog/wp-content/themes/twentyfifteen/style.css?ver=20230328' media='all' /> <link rel='stylesheet' id='twentyfifteen-block-style-css' href='https://www.generalist.org.uk/blog/wp-content/themes/twentyfifteen/css/blocks.css?ver=20230122' media='all' />   <link rel='stylesheet' id='wpgeshi-wp-geshi-highlight-css' href='https://www.generalist.org.uk/blog/wp-content/plugins/wp-geshi-highlight/wp-geshi-highlight.css?ver=6.4.5' media='all' /> <script src="https://www.generalist.org.uk/blog/wp-includes/js/jquery/jquery.min.js?ver=3.7.1" id="jquery-core-js"></script> <script src="https://www.generalist.org.uk/blog/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.4.1" id="jquery-migrate-js"></script> <link rel="https://api.w.org/" href="https://www.generalist.org.uk/blog/wp-json/" /><link rel="alternate" type="application/json" href="https://www.generalist.org.uk/blog/wp-json/wp/v2/tags/81" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://www.generalist.org.uk/blog/xmlrpc.php?rsd" /> <meta name="generator" content="WordPress 6.4.5" /> </head> <body class="archive tag tag-data tag-81 wp-embed-responsive"> <div id="page" class="hfeed site"> <a class="skip-link screen-reader-text" href="#content"> Skip to content </a> <div id="sidebar" class="sidebar"> <header id="masthead" class="site-header"> <div class="site-branding"> <p class="site-title"><a href="https://www.generalist.org.uk/blog/" rel="home">Andrew Gray</a></p> <button class="secondary-toggle">Menu and widgets</button> </div> </header> <div id="secondary" class="secondary"> <div id="widget-area" class="widget-area" role="complementary"> <aside id="recent-posts-3" class="widget widget_recent_entries"> <h2 class="widget-title">Recent Posts</h2><nav aria-label="Recent Posts"> <ul> <li> <a href="https://www.generalist.org.uk/blog/2024/caffenol/">Caffenol</a> <span class="post-date">January 11, 2024</span> </li> <li> <a href="https://www.generalist.org.uk/blog/2023/gender-and-blps-on-wikipedia-redux/">Gender and BLPs on Wikipedia, redux</a> <span class="post-date">August 2, 2023</span> </li> <li> <a href="https://www.generalist.org.uk/blog/2023/on-chatgpt/">on chatGPT</a> <span class="post-date">April 10, 2023</span> </li> <li> <a href="https://www.generalist.org.uk/blog/2021/lee-of-portrush-a-timeline/">Lee of Portrush: a timeline</a> <span class="post-date">September 18, 2021</span> </li> <li> <a href="https://www.generalist.org.uk/blog/2020/industrial-library/">Industrial Library</a> <span class="post-date">January 9, 2020</span> </li> </ul> </nav></aside><aside id="tag_cloud-3" class="widget widget_tag_cloud"><h2 class="widget-title">Tags</h2><nav aria-label="Tags"><div class="tagcloud"><ul class='wp-tag-cloud' role='list'> <li><a href="https://www.generalist.org.uk/blog/tags/administrivia/" class="tag-cloud-link tag-link-88 tag-link-position-1" style="font-size: 10.290909090909pt;" aria-label="administrivia (2 items)">administrivia</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/antarctica/" class="tag-cloud-link tag-link-72 tag-link-position-2" style="font-size: 10.290909090909pt;" aria-label="antarctica (2 items)">antarctica</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/archives/" class="tag-cloud-link tag-link-15 tag-link-position-3" style="font-size: 10.290909090909pt;" aria-label="archives (2 items)">archives</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/astronomy/" class="tag-cloud-link tag-link-18 tag-link-position-4" style="font-size: 11.818181818182pt;" aria-label="astronomy (3 items)">astronomy</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/books/" class="tag-cloud-link tag-link-9 tag-link-position-5" style="font-size: 16.272727272727pt;" aria-label="books (8 items)">books</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/british-library/" class="tag-cloud-link tag-link-63 tag-link-position-6" style="font-size: 10.290909090909pt;" aria-label="british library (2 items)">british library</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/campaigning/" class="tag-cloud-link tag-link-76 tag-link-position-7" style="font-size: 8pt;" aria-label="campaigning (1 item)">campaigning</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/canadian-collection/" class="tag-cloud-link tag-link-71 tag-link-position-8" style="font-size: 10.290909090909pt;" aria-label="canadian collection (2 items)">canadian collection</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/commonplace/" class="tag-cloud-link tag-link-56 tag-link-position-9" style="font-size: 13.090909090909pt;" aria-label="commonplace (4 items)">commonplace</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/commons/" class="tag-cloud-link tag-link-75 tag-link-position-10" style="font-size: 10.290909090909pt;" aria-label="commons (2 items)">commons</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/data/" class="tag-cloud-link tag-link-81 tag-link-position-11" style="font-size: 15.636363636364pt;" aria-label="data (7 items)">data</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/digitisation/" class="tag-cloud-link tag-link-103 tag-link-position-12" style="font-size: 8pt;" aria-label="digitisation (1 item)">digitisation</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/economics/" class="tag-cloud-link tag-link-12 tag-link-position-13" style="font-size: 16.272727272727pt;" aria-label="economics (8 items)">economics</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/election/" class="tag-cloud-link tag-link-39 tag-link-position-14" style="font-size: 19.2pt;" aria-label="election (14 items)">election</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/finance/" class="tag-cloud-link tag-link-47 tag-link-position-15" style="font-size: 11.818181818182pt;" aria-label="finance (3 items)">finance</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/freedom-of-information/" class="tag-cloud-link tag-link-31 tag-link-position-16" style="font-size: 10.290909090909pt;" aria-label="freedom of information (2 items)">freedom of information</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/history/" class="tag-cloud-link tag-link-21 tag-link-position-17" style="font-size: 22pt;" aria-label="history (24 items)">history</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/idiocy/" class="tag-cloud-link tag-link-35 tag-link-position-18" style="font-size: 11.818181818182pt;" aria-label="idiocy (3 items)">idiocy</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/internet/" class="tag-cloud-link tag-link-22 tag-link-position-19" style="font-size: 11.818181818182pt;" aria-label="internet (3 items)">internet</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/journalism/" class="tag-cloud-link tag-link-24 tag-link-position-20" style="font-size: 13.090909090909pt;" aria-label="journalism (4 items)">journalism</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/law/" class="tag-cloud-link tag-link-51 tag-link-position-21" style="font-size: 10.290909090909pt;" aria-label="law (2 items)">law</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/lee-of-portrush/" class="tag-cloud-link tag-link-95 tag-link-position-22" style="font-size: 10.290909090909pt;" aria-label="lee of portrush (2 items)">lee of portrush</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/legislation/" class="tag-cloud-link tag-link-79 tag-link-position-23" style="font-size: 8pt;" aria-label="legislation (1 item)">legislation</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/libraries/" class="tag-cloud-link tag-link-14 tag-link-position-24" style="font-size: 10.290909090909pt;" aria-label="libraries (2 items)">libraries</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/maps/" class="tag-cloud-link tag-link-50 tag-link-position-25" style="font-size: 10.290909090909pt;" aria-label="maps (2 items)">maps</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/media/" class="tag-cloud-link tag-link-23 tag-link-position-26" style="font-size: 10.290909090909pt;" aria-label="media (2 items)">media</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/mps/" class="tag-cloud-link tag-link-102 tag-link-position-27" style="font-size: 8pt;" aria-label="mps (1 item)">mps</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/open-access/" class="tag-cloud-link tag-link-77 tag-link-position-28" style="font-size: 10.290909090909pt;" aria-label="open access (2 items)">open access</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/oxford/" class="tag-cloud-link tag-link-20 tag-link-position-29" style="font-size: 11.818181818182pt;" aria-label="oxford (3 items)">oxford</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/parliament/" class="tag-cloud-link tag-link-100 tag-link-position-30" style="font-size: 10.290909090909pt;" aria-label="parliament (2 items)">parliament</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/photography/" class="tag-cloud-link tag-link-19 tag-link-position-31" style="font-size: 20.981818181818pt;" aria-label="photography (20 items)">photography</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/politics/" class="tag-cloud-link tag-link-8 tag-link-position-32" style="font-size: 21.236363636364pt;" aria-label="politics (21 items)">politics</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/publishing/" class="tag-cloud-link tag-link-27 tag-link-position-33" style="font-size: 16.909090909091pt;" aria-label="publishing (9 items)">publishing</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/recipes/" class="tag-cloud-link tag-link-6 tag-link-position-34" style="font-size: 13.090909090909pt;" aria-label="recipes (4 items)">recipes</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/remembrance/" class="tag-cloud-link tag-link-53 tag-link-position-35" style="font-size: 13.090909090909pt;" aria-label="remembrance (4 items)">remembrance</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/research/" class="tag-cloud-link tag-link-87 tag-link-position-36" style="font-size: 10.290909090909pt;" aria-label="research (2 items)">research</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/scotland/" class="tag-cloud-link tag-link-80 tag-link-position-37" style="font-size: 8pt;" aria-label="scotland (1 item)">scotland</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/security/" class="tag-cloud-link tag-link-28 tag-link-position-38" style="font-size: 10.290909090909pt;" aria-label="security (2 items)">security</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/software/" class="tag-cloud-link tag-link-46 tag-link-position-39" style="font-size: 10.290909090909pt;" aria-label="software (2 items)">software</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/statistics/" class="tag-cloud-link tag-link-25 tag-link-position-40" style="font-size: 16.909090909091pt;" aria-label="statistics (9 items)">statistics</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/travel/" class="tag-cloud-link tag-link-34 tag-link-position-41" style="font-size: 11.818181818182pt;" aria-label="travel (3 items)">travel</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/wikidata/" class="tag-cloud-link tag-link-78 tag-link-position-42" style="font-size: 16.272727272727pt;" aria-label="wikidata (8 items)">wikidata</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/wikimedia/" class="tag-cloud-link tag-link-74 tag-link-position-43" style="font-size: 13.090909090909pt;" aria-label="wikimedia (4 items)">wikimedia</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/wikipedia/" class="tag-cloud-link tag-link-26 tag-link-position-44" style="font-size: 21.745454545455pt;" aria-label="wikipedia (23 items)">wikipedia</a></li> <li><a href="https://www.generalist.org.uk/blog/tags/ww1/" class="tag-cloud-link tag-link-65 tag-link-position-45" style="font-size: 13.090909090909pt;" aria-label="ww1 (4 items)">ww1</a></li> </ul> </div> </nav></aside><aside id="archives-3" class="widget widget_archive"><h2 class="widget-title">Archives</h2> <label class="screen-reader-text" for="archives-dropdown-3">Archives</label> <select id="archives-dropdown-3" name="archive-dropdown"> <option value="">Select Month</option> <option value='https://www.generalist.org.uk/blog/2024/01/'> January 2024  (1)</option> <option value='https://www.generalist.org.uk/blog/2023/08/'> August 2023  (1)</option> <option value='https://www.generalist.org.uk/blog/2023/04/'> April 2023  (1)</option> <option value='https://www.generalist.org.uk/blog/2021/09/'> September 2021  (1)</option> <option value='https://www.generalist.org.uk/blog/2020/01/'> January 2020  (1)</option> <option value='https://www.generalist.org.uk/blog/2019/05/'> May 2019  (1)</option> <option value='https://www.generalist.org.uk/blog/2019/03/'> March 2019  (1)</option> <option value='https://www.generalist.org.uk/blog/2019/02/'> February 2019  (1)</option> <option value='https://www.generalist.org.uk/blog/2018/11/'> November 2018  (1)</option> <option value='https://www.generalist.org.uk/blog/2018/10/'> October 2018  (1)</option> <option value='https://www.generalist.org.uk/blog/2017/09/'> September 2017  (1)</option> <option value='https://www.generalist.org.uk/blog/2017/01/'> January 2017  (2)</option> <option value='https://www.generalist.org.uk/blog/2016/08/'> August 2016  (1)</option> <option value='https://www.generalist.org.uk/blog/2016/04/'> April 2016  (1)</option> <option value='https://www.generalist.org.uk/blog/2016/02/'> February 2016  (3)</option> <option value='https://www.generalist.org.uk/blog/2016/01/'> January 2016  (2)</option> <option value='https://www.generalist.org.uk/blog/2015/12/'> December 2015  (1)</option> <option value='https://www.generalist.org.uk/blog/2015/11/'> November 2015  (2)</option> <option value='https://www.generalist.org.uk/blog/2015/10/'> October 2015  (1)</option> <option value='https://www.generalist.org.uk/blog/2015/08/'> August 2015  (1)</option> <option value='https://www.generalist.org.uk/blog/2015/02/'> February 2015  (2)</option> <option value='https://www.generalist.org.uk/blog/2015/01/'> January 2015  (1)</option> <option value='https://www.generalist.org.uk/blog/2014/11/'> November 2014  (2)</option> <option value='https://www.generalist.org.uk/blog/2014/10/'> October 2014  (1)</option> <option value='https://www.generalist.org.uk/blog/2014/09/'> September 2014  (3)</option> <option value='https://www.generalist.org.uk/blog/2014/08/'> August 2014  (1)</option> <option value='https://www.generalist.org.uk/blog/2013/12/'> December 2013  (1)</option> <option value='https://www.generalist.org.uk/blog/2013/08/'> August 2013  (1)</option> <option value='https://www.generalist.org.uk/blog/2013/06/'> June 2013  (2)</option> <option value='https://www.generalist.org.uk/blog/2013/05/'> May 2013  (1)</option> <option value='https://www.generalist.org.uk/blog/2013/04/'> April 2013  (1)</option> <option value='https://www.generalist.org.uk/blog/2013/02/'> February 2013  (3)</option> <option value='https://www.generalist.org.uk/blog/2012/12/'> December 2012  (1)</option> <option value='https://www.generalist.org.uk/blog/2012/11/'> November 2012  (1)</option> <option value='https://www.generalist.org.uk/blog/2012/10/'> October 2012  (1)</option> <option value='https://www.generalist.org.uk/blog/2012/02/'> February 2012  (2)</option> <option value='https://www.generalist.org.uk/blog/2011/11/'> November 2011  (3)</option> <option value='https://www.generalist.org.uk/blog/2011/07/'> July 2011  (1)</option> <option value='https://www.generalist.org.uk/blog/2011/06/'> June 2011  (2)</option> <option value='https://www.generalist.org.uk/blog/2011/05/'> May 2011  (2)</option> <option value='https://www.generalist.org.uk/blog/2011/02/'> February 2011  (1)</option> <option value='https://www.generalist.org.uk/blog/2011/01/'> January 2011  (2)</option> <option value='https://www.generalist.org.uk/blog/2010/12/'> December 2010  (2)</option> <option value='https://www.generalist.org.uk/blog/2010/11/'> November 2010  (6)</option> <option value='https://www.generalist.org.uk/blog/2010/10/'> October 2010  (5)</option> <option value='https://www.generalist.org.uk/blog/2010/09/'> September 2010  (1)</option> <option value='https://www.generalist.org.uk/blog/2010/08/'> August 2010  (4)</option> <option value='https://www.generalist.org.uk/blog/2010/07/'> July 2010  (2)</option> <option value='https://www.generalist.org.uk/blog/2010/06/'> June 2010  (4)</option> <option value='https://www.generalist.org.uk/blog/2010/05/'> May 2010  (11)</option> <option value='https://www.generalist.org.uk/blog/2010/04/'> April 2010  (10)</option> <option value='https://www.generalist.org.uk/blog/2010/03/'> March 2010  (9)</option> <option value='https://www.generalist.org.uk/blog/2010/02/'> February 2010  (8)</option> <option value='https://www.generalist.org.uk/blog/2010/01/'> January 2010  (8)</option> <option value='https://www.generalist.org.uk/blog/2009/12/'> December 2009  (3)</option> <option value='https://www.generalist.org.uk/blog/2009/11/'> November 2009  (7)</option> </select> <script> (function() { var dropdown = document.getElementById( "archives-dropdown-3" ); function onSelectChange() { if ( dropdown.options[ dropdown.selectedIndex ].value !== '' ) { document.location.href = this.options[ this.selectedIndex ].value; } } dropdown.onchange = onSelectChange; })(); </script> </aside><aside id="block-6" class="widget widget_block"><h2 class="widget-title">Elsewhere</h2> <ul><li><a href="https://twitter.com/generalising" data-type="URL" data-id="https://twitter.com/generalising" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a href="https://mastodon.flooey.org/web/@generalising" data-type="URL" data-id="https://mastodon.flooey.org/web/@generalising" target="_blank" rel="me noreferrer noopener">Mastodon</a></li></ul></aside> </div> </div> </div> <div id="content" class="site-content"> <section id="primary" class="content-area"> <main id="main" class="site-main"> <header class="page-header"> <h1 class="page-title">Tag: <span>data</span></h1> </header> <article id="post-1714" class="post-1714 post type-post status-publish format-standard hentry category-andrew tag-data tag-statistics tag-wikidata tag-wikipedia"> <header class="entry-header"> <h2 class="entry-title"><a href="https://www.generalist.org.uk/blog/2023/gender-and-blps-on-wikipedia-redux/" rel="bookmark">Gender and BLPs on Wikipedia, redux</a></h2> </header> <div class="entry-content"> <p>Back in 2019 I <a rel="noreferrer noopener" href="https://www.generalist.org.uk/blog/2019/gender-and-deletion-on-wikipedia/" target="_blank">generated</a><a href="https://www.generalist.org.uk/blog/2019/gender-and-deletion-on-wikipedia/"> </a><a rel="noreferrer noopener" href="https://www.generalist.org.uk/blog/2019/gender-and-deletion-on-wikipedia/" target="_blank">some</a><a href="https://www.generalist.org.uk/blog/2019/gender-and-deletion-on-wikipedia/"> data</a> on how the English Wikipedia’s biographies of living people (BLPs) broke down by gender, and how that intersected with creation and deletion rates. The headline figures were that:</p> <ul> <li>There was a significant difference between the gender split for all biographies, at 17.8% women – and for biographies of living people (BLPs), 22.7%.</li> <li>In 2009, around 20% of <em>existing</em> BLPs were on women. As time went on, the average <em>share</em> of BLPs increased slowly, by perhaps a quarter of a percentage point per year.</li> <li>In 2009, around 20% of <em>newly created</em> BLPs were on women. In about 2012, this kicked up a gear, rising above the long term average – first to about 25%, peaking around 33% before falling back a little.</li> <li>BLP articles on women were more likely to be nominated for deletion <em>until about 2017</em>, when the effect disappeared.</li> </ul> <p>One thing that was raised during the subsequent discussion was that a lot of the skew by gender was potentially linked to subject area – I was able to identify that for athletes (defined broadly, all sports players) the articles were much more likely to be men. I didn’t investigate this too much, though. Last week, I was <a rel="noreferrer noopener" href="https://en.wikipedia.org/wiki/Wikipedia_talk:WikiProject_Women_in_Red#Topics_that_account_for_a_disproportionate_amount_of_the_gender_gap_in_biographies" target="_blank">reminded about this</a>, and I’ve been looking at the numbers again. It brought up two interesting divergences.</p> <h3 class="wp-block-heading">Please don’t make me read all this</h3> <p>Okay – but you’ll miss the graphs. In summary:</p> <p><em>English Wikipedia has more women in recent cohorts (about ~25% of living people born since the seventies) and there are far more men among athletes. Since the athletes make up a staggeringly high amount of articles among younger subjects, the gender split among non-athletes is much more balanced – a little under a third overall, but breaking 50% female among the younger cohorts.</em></p> <p>Still with me? Let’s start. Sorry about the spoilers.</p> <h3 class="wp-block-heading">Time and tide</h3> <p>The first phenomenon is very straightforward: while the overall percentage across all people is around 25% women, how that is <em>distributed</em> over time varies. In general, there is a steady rise until about the 1970s; <strong>for those born from the 1970s onwards, the generation who are currently in their active working lives, the level is relatively stable at around 25% women.</strong></p> <p>The exception is those born in the 1920s (where it sits at 26%) – this is presumably affected by the fact that at this point, female life expectancy is significantly higher than male, and so the proportion of women begins to rise as a result.</p> <figure class="wp-block-image size-full"><a href="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-people-on-Wikipedia-female1.png"><img fetchpriority="high" decoding="async" width="600" height="371" src="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-people-on-Wikipedia-female1.png" alt="" class="wp-image-1730" srcset="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-people-on-Wikipedia-female1.png 600w, https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-people-on-Wikipedia-female1-300x186.png 300w" sizes="(max-width: 600px) 100vw, 600px" /></a></figure> <p>One surprising outcome, however, is that <strong>the share of living people with no recorded age (green) is much more female than the average</strong>. This is a <em>large</em> cohort – there are in fact slightly more women in it than in any individual decade. I believe that it skews young – in other words, were this information known, it would increase the share of women in recent decades – but it is hard to find a way to confirm this. This issue is discussed in more detail below.</p> <p>(Those born in the 2010s/20s and in the 1900s/10s are omitted – the four groups have a total of 175 articles, while the cohorts shown range from 5,000 to 170,000 – but the levels are around 50%. This is likely due to life expectancy in the oldest cohorts, and the fact that the people in the youngest cohorts are mostly notable at this point as being “the child of someone famous” – which you would broadly expect to be independent of gender.)</p> <p>The percentages shown here are of the total male + female articles, but it is also possible to calculate the share of people who have a recorded gender that is not male/female. These show a very striking rise over time, though it should be cautioned that the absolute numbers are small – the largest single cohort is the 1980s with 345 people out of 170,000.</p> <figure class="wp-block-image size-full"><a href="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-people-on-Wikipedia-not-M_F.png"><img decoding="async" width="600" height="371" src="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-people-on-Wikipedia-not-M_F.png" alt="" class="wp-image-1723" srcset="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-people-on-Wikipedia-not-M_F.png 600w, https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-people-on-Wikipedia-not-M_F-300x186.png 300w" sizes="(max-width: 600px) 100vw, 600px" /></a></figure> <h3 class="wp-block-heading">Sports by the numbers</h3> <p>The original question was to look at what the effect of athlete articles is on the overall totals. It turns out… very striking. </p> <figure class="wp-block-image size-full is-resized"><a href="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Living-people-on-Wikipedia-by-sports-status.png"><img decoding="async" src="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Living-people-on-Wikipedia-by-sports-status.png" alt="" class="wp-image-1726" width="600" height="371" srcset="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Living-people-on-Wikipedia-by-sports-status.png 600w, https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Living-people-on-Wikipedia-by-sports-status-300x186.png 300w" sizes="(max-width: 600px) 100vw, 600px" /></a></figure> <p>A few things are immediately apparent. The first is that <strong>the share of athletes is very substantial</strong> – it reflects only around a quarter of people born in the 1950s, but 85-90% of people born in the 1990s/2000s. </p> <p>The second is that <strong>those athletes are overwhelmingly men</strong> – among the 1950s cohort, only about 10% of those athletes are female, and even by recent years it is only around 20%. This means that if we look purely at the non-athlete articles, the gender split becomes a lot more balanced. </p> <figure class="wp-block-image size-full"><a href="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-non-athletes-on-Wikipedia-female.png"><img loading="lazy" decoding="async" width="600" height="371" src="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-non-athletes-on-Wikipedia-female.png" alt="" class="wp-image-1727" srcset="https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-non-athletes-on-Wikipedia-female.png 600w, https://www.generalist.org.uk/blog/wp-content/uploads/2023/08/Gender-split-for-living-non-athletes-on-Wikipedia-female-300x186.png 300w" sizes="(max-width: 600px) 100vw, 600px" /></a></figure> <p>Across all articles, it is around 32% female. But <strong>among living <strong>non-athlete</strong>s, born since 1990, the gender balance is over 50% female.</strong></p> <p>This is a really amazing figure. I don’t think I ever particularly expected to see a gender analysis on Wikipedia that would break 50%. Granted, the absolute numbers involved are low – as is apparent from the previous graph, “non-athletes born in the 1990s” is around 22,000 people, and “born in the 2000s” is as low as 2,500 – but it’s a pretty solid trend and the total numbers for the earlier decades are definitely large enough for it to be no anomaly.</p> <p>(Eagle-eyed readers will note that these do not quite align with the numbers in the original linked discussion – those were a couple of points lower in recent decades. I have not quite worked out why, but I think this was an error in the earlier queries; possibly it was counting redirects?)</p> <p>One last detail to note: the “date missing” cohort comes out over 90% <em>non</em>-athletes. Presumably this is because their exact age is often significant and linked in to eg when they start professional sports, so it’s easily publicly available. </p> <h3 class="wp-block-heading">Methodology: the thousand word footnote</h3> <p>Feel free to let your eyes glaze over now.</p> <p>These numbers were constructed mostly using the <a rel="noreferrer noopener" href="https://petscan.wmflabs.org/" data-type="URL" data-id="https://petscan.wmflabs.org/" target="_blank">petscan</a> tool, and leveraging data from both English Wikipedia and Wikidata. From Wikipedia, we have a robust categorisation system for year/decade of birth, and for whether someone is a living person. From Wikidata, we have fairly comprehensive gender data, which Wikipedia doesn’t know about. (It also has dates of birth, but it is more efficient to use WP categories here). So it is straightforward to produce intersection queries like “all living people marked as 1920s births and marked as female” (<a href="https://petscan.wmflabs.org/?outlinks_any=&namespace_conversion=keep&cb_labels_any_l=1&labels_no=&search_wiki=&common_wiki=auto&show_soft_redirects=both&depth=0&sitelinks_no=&show_redirects=no&min_redlink_count=1&categories=Living+people%0D%0A1920s+births%7C1&sparql=select+*+where+%7B+%3Fitem+wdt%3AP21+wd%3AQ6581072+%7D&output_compatability=catscan&templates_any=&active_tab=tab_other_sources&langs_labels_no=&maxlinks=&sitelinks_yes=&links_to_all=&cb_labels_yes_l=1&edits%5Bflagged%5D=both&interface_language=en&search_max_results=500&templates_yes=&format=html&show_disambiguation_pages=no&language=en&referrer_name=&negcats=&project=wikipedia&combination=subset&ns%5B0%5D=1&sortorder=ascending&edits%5Bbots%5D=both&min_sitelink_count=&page_image=any&max_age=&edits%5Banons%5D=both&cb_labels_no_l=1&subpage_filter=either">report</a>). Note that this is crunching a lot of data – don’t be surprised if queries take a minute or two to run or occasionally time out.</p> <p>To my surprise, the report for “living people known to be female” initially produced a reliable figure, but one for “living people known to be male” produced a figure that was an undercount. (I could validate this by checking against some small categories where I could run a report listing the gender of every item). The root cause seemed to be a timeout in the Wikidata query – I was originally looking for<em> { ?item wdt:P31 wd:Q5 . wdt:P21 wd:Q6581097 }</em> – items known to be human with gender male. Tweaking this to be simply <em>{ ?item wdt:P21 wd:Q6581097 }</em> – items with gender male – produced a reliable figure. Similarly, we had the same issue when trying to get a total for all items with reported gender – simply <em>{ ?item wdt:P21 ?val }</em> works.</p> <p>Percentages are calculated as percentage of the number of articles identified as (male + female), rather than of all BLPs with a recorded gender value or simply of all BLPs. There are good arguments for either of the first two, but the former is simpler (some of my “any recorded gender value” queries timed out) and also consistent with the 2019 analysis. </p> <p>A thornier problem comes from the sports element. There are a number of potential ways we could determine “sportiness”. The easiest option would be to use Wikidata occupation and look for something that indicates their occupation is some form of athlete, or that indicates a sport being played. The problem is that this is too all-encompassing, and would give us people who <em>played</em> sports but for whom it is not their main claim to fame. An alternative is to use the Wikipedia article categorisation hierarchy, but this is very complex and deep, making the queries very difficult to work with. The category hierarchy includes a number of surprise crosslinks and loops, meaning that deep queries tend to get very confusing results, or just time out.</p> <p>The approach I eventually went with was to use Wikipedia’s infoboxes – the little standardised box on the top right of a page. There are a wide range of distinct infobox templates tailored to specific fields; each article usually only displays one, but can embed elements of others to bring in secondary data. If we look for articles using one of the 77(!) distinct sports infoboxes (<a rel="noreferrer noopener" href="https://petscan.wmflabs.org/?labels_any=&labels_yes=&regexp_filter=&search_query=&templates_no=&manual_list=&links_to_any=&categories=Living+people&output_limit=&labels_no=&ores_type=any&edits%5Bflagged%5D=both&outlinks_any=&namespace_conversion=keep&psid=25494250&maxlinks=&cb_labels_no_l=1&templates_any=Infobox+F1+driver%0D%0AInfobox+professional+wrestler%0D%0AInfobox+tennis+biography%0D%0AInfobox+boxer%0D%0AInfobox+Canadian+Football+League+biography%0D%0AInfobox+figure+skater%0D%0AInfobox+snooker+player%0D%0AInfobox+Champ+Car+driver%0D%0AInfobox+sumo+wrestler%0D%0AInfobox+Gaelic+games+player%0D%0AInfobox+basketball+biography%0D%0AInfobox+ice+hockey+biography%0D%0AInfobox+college+coach%0D%0AInfobox+bodybuilder%0D%0AInfobox+British+Touring+Car+Championship+record%0D%0AInfobox+rugby+biography%0D%0AInfobox+rugby+league+biography%0D%0AInfobox+lacrosse+player%0D%0AInfobox+V8+Supercar+record%0D%0AInfobox+darts+player%0D%0AInfobox+NASCAR+driver%0D%0AInfobox+field+hockey+player%0D%0AInfobox+sports+announcer+details%0D%0AInfobox+volleyball+biography%0D%0AInfobox+NCAA+athlete%0D%0AInfobox+WRC+driver%0D%0AInfobox+Gaelic+games+manager%0D%0AInfobox+cyclist%0D%0AInfobox+badminton+player%0D%0AInfobox+alpine+ski+racer%0D%0AInfobox+go+player%0D%0AInfobox+baseball+biography%0D%0AInfobox+horseracing+personality%0D%0AInfobox+racing+driver%0D%0AInfobox+skier%0D%0AInfobox+Le+Mans+driver%0D%0AInfobox+chess+biography%0D%0AInfobox+speed+skater%0D%0AInfobox+speedway+rider%0D%0AInfobox+swimmer%0D%0AInfobox+NFL+biography%0D%0AInfobox+cricketer%0D%0AInfobox+martial+artist%0D%0AInfobox+amateur+wrestler%0D%0AInfobox+AFL+biography%0D%0AInfobox+college+football+player%0D%0AInfobox+table+tennis+player%0D%0AInfobox+Motocross+rider%0D%0AInfobox+motorcycle+rider%0D%0AInfobox+netball+biography%0D%0AInfobox+handball+biography%0D%0AInfobox+gymnast%0D%0AInfobox+sportsperson%0D%0AInfobox+climber%0D%0AInfobox+boxer+%28amateur%29%0D%0AInfobox+golfer%0D%0AInfobox+football+official%0D%0AInfobox+squash+player%0D%0AInfobox+surfer%0D%0AInfobox+professional+bowler%0D%0AInfobox+pelotari%0D%0AInfobox+curler%0D%0AInfobox+fencer%0D%0AInfobox+bullfighting+career%0D%0AInfobox+mountaineer%0D%0AInfobox+tennis+player+season%0D%0AInfobox+racing+driver+series+section%0D%0AInfobox+sailor%0D%0AInfobox+biathlete%0D%0AInfobox+sport+wrestler%0D%0AInfobox+football+biography%0D%0AInfobox+bandy+biography%0D%0AInfobox+checkers+biography%0D%0AInfobox+water+polo+biography%0D%0AInfobox+sportsperson%2FWikidata%0D%0AInfobox+pool+player%0D%0AInfobox+judoka&templates_yes=&sitelinks_no=&cb_labels_any_l=1&search_max_results=500&source_combination=&language=en&sortby=none&langs_labels_no=&common_wiki_other=&cb_labels_yes_l=1&langs_labels_any=&edits%5Banons%5D=both&project=wikipedia&links_to_all=&depth=0&referrer_name=&wpiu=any&negcats=&combination=subset&ns%5B0%5D=1&min_sitelink_count=&referrer_url=&active_tab=tab_templates_n_links&interface_language=en&before=&wikidata_prop_item_use=" target="_blank">report</a>), we can conclude they probably had a significant sporting career. An article that does not contain one can be inferred to not have a sporting background.</p> <p>But then we need to consider people with significant sports <em>and</em> non-sports careers. For example, the biographies of both <a rel="noreferrer noopener" href="https://en.wikipedia.org/wiki/Sebastian_Coe" data-type="URL" data-id="https://en.wikipedia.org/wiki/Sebastian_Coe" target="_blank">Seb Coe</a> and <a rel="noreferrer noopener" href="https://en.wikipedia.org/wiki/Tanni_Grey-Thompson" data-type="URL" data-id="https://en.wikipedia.org/wiki/Tanni_Grey-Thompson" target="_blank">Tanni Grey-Thompson</a> use the “infobox officeholder” to reflect their careers in Parliament being more recent, but it is set up to embed a sports infobox towards the end. This would entail them being counted as athletes by our infobox method. This is probably correct for <em>those</em> two, but there are no doubt people out there where we would draw the line differently. (To stay in the UK political theme: how about <a href="https://en.wikipedia.org/wiki/Henry_McLeish" data-type="URL" data-id="https://en.wikipedia.org/wiki/Henry_McLeish">Henry McLeish</a>? His athletic career on its own would probably <em>just</em> qualify for a Wikipedia biography, but it is, perhaps, a bit of a footnote compared to being First Minister…)</p> <p>So, here is another complication. How reliable is our assumption that an athlete has a sports infobox, and that non-athletes don’t? If it’s broadly true, great, our numbers hold up. If it’s not, and if it’s not in some kind of systematic way, there might be a more complex skew. I <em>believe</em> that for modern athletes, it’s reasonably safe to assume that infoboxes are nearly ubiquitous; there are groups of articles where they’re less common, but this isn’t one of them. However, I can’t say for sure; it’s not an area I’ve worked intensively in.</p> <p>Finally, we have the issue of dates. We’ve based the calculation on Wikipedia categories. Wikipedia birth/death categories are pretty reliably used where that data is known. However, about 150k (14%) of our BLP articles are marked “year of birth unknown”, and these are disproportionately female (35.4%). </p> <p>What effect do these factors have?</p> <p>Counting the stats as percentage of M+F rather than percentage of all people with recorded gender could be argued either way, but the numbers involved are quite low and do not change the overall pattern of the results.</p> <p>The infobox question is more complicated. It is possible that it is meaning we are not picking up all athletes because they do not have infoboxes. On the other hand, it is possible that it is meaning we are being more expansive in counting people as athletes because they have a “secondary” infobox along the line of Coe & Grey-Thompson above. The problem there is defining where we draw the line, and what level of “other significance” stops someone being counted. That feels like a very subjective threshold and hard to test for automatically. It is certainly a more conservative test than a Wikidata-based one, at least.</p> <p>And for dates, hmm. We know that the articles that do not report an age are disproportionately female (35% vs the BLP average of 25%), but also that they are <em>even</em> more disproportionately “not athletes” (7% athletes vs the BLP average of 43%). There are also a lot of articles that don’t report an age; around 14% of all BLPs.</p> <p>This one probably introduces the biggest question mark here. Depending on how that 14% break down, it could change the totals for the year-by-year cohorts; but there’s not really much we can do at the moment to work that out.</p> <p>Anecdotally, I suspect that they are more likely to skew younger rather than being evenly distributed over time, but there is very little to go on here. However, I feel it is unlikely they would be distributed in such a way as to <em>counteract</em> the overall conclusions – this would require, for example, the female ones being predominantly shifted into older groups and the male ones into younger groups. It’s possible, but I don’t see an obvious mechanism to cause that.</p> <p><em>[Edit 4/8/23 – tweaked to confirm that these are English Wikipedia only figures, after a reminder from <a rel="noreferrer noopener" href="https://hildabastian.wordpress.com/2023/08/04/some-knowledge-gaps-are-no-coincidence/" data-type="URL" data-id="https://hildabastian.wordpress.com/2023/08/04/some-knowledge-gaps-are-no-coincidence/" target="_blank">Hilda</a>. I would be very interested in seeing similar data for other projects, but the methodology might be tricky to translate – eg French and German do not have an equivalent category for indexing living people, and different projects may have quite different approaches for applying infoboxes.]</em></p> </div> <footer class="entry-footer"> <span class="posted-on"><span class="screen-reader-text">Posted on </span><a href="https://www.generalist.org.uk/blog/2023/gender-and-blps-on-wikipedia-redux/" rel="bookmark"><time class="entry-date published" datetime="2023-08-02T23:36:52+01:00">August 2, 2023</time><time class="updated" datetime="2023-08-04T18:14:40+01:00">August 4, 2023</time></a></span><span class="cat-links"><span class="screen-reader-text">Categories </span><a href="https://www.generalist.org.uk/blog/authors/andrew/" rel="category tag">Andrew</a></span><span class="tags-links"><span class="screen-reader-text">Tags </span><a href="https://www.generalist.org.uk/blog/tags/data/" rel="tag">data</a>, <a href="https://www.generalist.org.uk/blog/tags/statistics/" rel="tag">statistics</a>, <a href="https://www.generalist.org.uk/blog/tags/wikidata/" rel="tag">wikidata</a>, <a href="https://www.generalist.org.uk/blog/tags/wikipedia/" rel="tag">wikipedia</a></span><span class="comments-link"><a href="https://www.generalist.org.uk/blog/2023/gender-and-blps-on-wikipedia-redux/#comments">1 Comment<span class="screen-reader-text"> on Gender and BLPs on Wikipedia, redux</span></a></span> </footer> </article> <article id="post-1578" class="post-1578 post type-post status-publish format-standard hentry category-andrew tag-data tag-statistics tag-wikidata tag-wikipedia"> <header class="entry-header"> <h2 class="entry-title"><a href="https://www.generalist.org.uk/blog/2019/gender-and-deletion-on-wikipedia/" rel="bookmark">Gender and deletion on Wikipedia</a></h2> </header> <div class="entry-content"> <p>So, a really interesting question <a href="https://www.wikidata.org/wiki/Wikidata:Request_a_query#Question_about_BLPs">cropped up</a> this weekend:</p> <blockquote><p> I’m trying to find out how many biographies of living persons exist on the English Wikipedia, and what kind of data we have on them. In particular, I’m looking for the gender breakdown. I’d also like to know when they were created; average length; and whether they’ve been nominated for deletion.</p></blockquote> <p>This is, of course, something that’s being discussed a lot right now; there is a lot of emerging push-back against the <a href="https://www.nature.com/articles/d41586-018-05947-8">excellent work</a> being done to try and add more notable women to Wikipedia, and one particular deletion debate <a href="https://undark.org/2019/04/25/wikipedia-diversity-problem/">got a lot of attention</a> in the past few weeks, so it’s on everyone’s mind. And, instinctively, it seems plausible that there is a bias in the relative frequency of nomination for deletion – can we find if it’s there?</p> <p>My initial assumption was, huh, I don’t think we can do that with Wikidata. Then I went off and thought about it for a bit more, and realised we could get <i>most</i> of the way there of it with some inferences. Here’s the results, and how I got there. Thanks to <a href="https://www.wikidata.org/wiki/User:SlimVirgin">Sarah</a> for prompting the research!</p> <p>(If you want to get the tl;dr summary – yes, there is <b>some kind</b> of difference in the way older male vs female articles have been involved with the deletion process, but exactly what that indicates is not obvious without data I can’t get at. The difference seems to have <i>mostly</i> disappeared for articles created in the last couple of years.)</p> <h3>Statistics on the gender breakdown of BLPs</h3> <p>As of a snapshot of yesterday morning, 5 May 2019, the English Wikipedia had 906,720 articles identified as biographies of living people (BLPs for short). Of those, 697,402 were identified as male by Wikidata, 205,117 as female, 2464 had some other value for gender, 1220 didn’t have any value for gender (usually articles on <i>groups</i> of people, plus some not yet updated), and 517 simply didn’t have a connected Wikidata item (yet). Of those with known gender, it breaks down as 77.06% male, 22.67% female, and 0.27% some other value. (Because of the limits of the query, I didn’t try and break down those in any more detail.)</p> <p>This is, as noted, only articles about <i>living people</i>; across <a href="https://www.denelezh.org/gender-gap/?project=enwiki&sort=label">all 1,626,232 biographies in the English Wikipedia</a> with a gender known to Wikidata, it’s about 17.83% female, 82.13% male, and 0.05% some other value. I’ll be sticking to data on living people throughout this post, but it’s interesting to compare the historic information.</p> <p>So, how has that changed over time?</p> <p><a title="Andrew Gray [CC BY-SA 3.0 (https://creativecommons.org/licenses/by-sa/3.0)], via Wikimedia Commons" href="https://commons.wikimedia.org/wiki/File:BLPs_by_gender_and_date_of_creation.png"><img decoding="async" alt="BLPs by gender and date of creation" src="https://upload.wikimedia.org/wikipedia/commons/4/42/BLPs_by_gender_and_date_of_creation.png" width="512"></a></p> <p>This graph shows all existing BLPs, broken down by gender and (approximately) when they were created. As can be seen, and as might be expected, the gap has closed a bit over time.</p> <p><a title="Andrew Gray [CC BY-SA 3.0 (https://creativecommons.org/licenses/by-sa/3.0)], via Wikimedia Commons" href="https://commons.wikimedia.org/wiki/File:Percentage_of_BLPs_which_are_female_over_time.png"><img decoding="async" alt="Percentage of BLPs which are female over time" src="https://upload.wikimedia.org/wikipedia/commons/c/c0/Percentage_of_BLPs_which_are_female_over_time.png" width="512"></a></p> <p>Looking at the ratio over time (expressed here as %age of total male+female), the relative share of female BLPs was ~20% in 2009. In late 2012, the rate of creation of female BLPs kicked up a gear, and from then on it’s been noticeably above the long-term average (almost hitting 33% in late 2017, but dropping back since then). This has driven the overall share steadily and continually upwards, now at 22.7% (as noted above).</p> <p>Now the second question, do the article lengths differ by gender? Indeed they do, by a small amount.</p> <p><a title="Andrew Gray [CC BY-SA 3.0 (https://creativecommons.org/licenses/by-sa/3.0)], via Wikimedia Commons" href="https://commons.wikimedia.org/wiki/File:BLPs_by_current_article_size_and_date_of_creation.png"><img decoding="async" alt="BLPs by current article size and date of creation" src="https://upload.wikimedia.org/wikipedia/commons/e/ea/BLPs_by_current_article_size_and_date_of_creation.png" width="512"></a></p> <p>Female BLPs created at any time since 2009 are slightly longer on average than male ones of similar age, with only a couple of brief exceptions; the gap may be widening over the past year but it’s maybe too soon to say for sure. Average difference is about 500 bytes or a little under 10% of mean article size – not dramatic but probably not trivial either. (Pre-2009 articles, not shown here, are about even on average)</p> <p>Note that this is raw bytesize – actual prose size will be smaller, particularly if an article is well-referenced; a single well-structured reference can be a few hundred characters. It’s also the <i>current</i> article size, not size at creation, hence why older articles tend to be longer – they’ve had more time to grow. It’s interesting to note that once they’re more than about five years old they seem to plateau in average length.</p> <p>Finally, the third question – have they been nominated for deletion? This was really interesting.</p> <p><a title="Andrew Gray [CC BY-SA 3.0 (https://creativecommons.org/licenses/by-sa/3.0)], via Wikimedia Commons" href="https://commons.wikimedia.org/wiki/File:Percentage_of_BLPs_which_have_previously_been_to_AFD,_by_date_of_creation_and_gender.png"><img decoding="async" alt="Percentage of BLPs which have previously been to AFD, by date of creation and gender" src="https://upload.wikimedia.org/wikipedia/commons/c/c2/Percentage_of_BLPs_which_have_previously_been_to_AFD%2C_by_date_of_creation_and_gender.png" width="512"></a></p> <p>So, first of all, some caveats. This only identifies articles which go through the structured “articles for deletion” (AFD) process – nomination, discussion, decision to keep or delete. (There are <a href="https://en.wikipedia.org/wiki/Wikipedia:Deletion_process">three deletion processes</a> on Wikipedia; the other two are more lightweight and do not show up in an easily traceable form). It also cannot specifically identify if <i>that exact page</i> was nominated for deletion, only that “an article with exactly the same page name has been nominated in the past” – but the odds are good they’re the same if there’s a match. It will miss out any where the article was renamed after the deletion discussion, and, most critically, <i>it will only see articles that survived deletion</i>. If they were deleted, I won’t be able to see them in this analysis, so there’s an obvious survivorship bias limiting what conclusions we can draw.</p> <p>Having said all that…</p> <p>Female BLPs created 2009-16 appear noticeably more likely than male BLPs of equivalent age to have been through a deletion discussion at some point in their lives (and, presumably, all have been kept). Since 2016, this has changed and the two groups are about even.</p> <p>Alongisde this, there is a corresponding drop-off in the number of articles created since 2016 which have associated deletion discussions. My tentative hypothesis is that articles created in the last few years are generally less likely to be nominated for deletion, perhaps because the growing use of things like the draft namespace (and associated reviews) means that articles are more robust when first published. Conversely, though, it’s possible that nominations continue at the same rate, but the deletion process is just more rigorous now and a higher proportion of those which are nominated get deleted (and so disappear from our data). We can’t tell.</p> <p>(One possible explanation that we can tentatively dismiss is age – an article can be nominated at any point in its lifespan so you would tend to expect a slowly increasing share over time, but I would expect the majority of deletion nominations come in the first weeks and then it’s pretty much evenly distributed after that. As such, the drop-off seems far too rapid to be explained by <i>just</i> article age.)</p> <p>What we don’t know is what the overall <i>nomination for deletion</i> rate, including deleted articles, looks like. From our data, it could be that pre-2016 male and female articles are nominated at equal rates but more male articles are deleted; or it could be that pre-2016 male and female articles are equally likely to get deleted, but the female articles are nominated more frequently than they should be. Either of these would cause the imbalance. I think this is very much the missing piece of data and I’d love to see any suggestions for how we can work it out – perhaps something like trying to estimate gender from the names of deleted articles?</p> <p><b>Update:</b> Magnus has <a href="http://magnusmanske.de/wordpress/?p=613">run some numbers on deleted pages</a>, doing exactly this – inferring gender from pagenames. Of those which were probably a person, ~2/3 had an inferred gender, and 23% of those were female. This is a remarkably similar figure to the analysis here (~23% of current BLPs female; ~26% of all BLPs which have survived a deletion debate female)</p> <p><b>So in conclusion</b>…</p> <ul> <li>We know the gender breakdown: skewed male, but growing slowly more balanced over time, and better for living people than historical ones.</li> <li>We know the article lengths; slightly longer for women than men for recent articles, about equal for those created a long time ago.</li> <li>We know that there is <i>something</i> different about the way male and female biographies created before ~2017 experience the deletion process, but we don’t have clear data to indicate exactly what is going on, and there are multiple potential explanations.</li> <li>We also know that deletion activity seems to be more balanced for articles in both groups created from ~2017 onwards, and that these also have a lower frequency of involvement with the deletion process than might have been expected. It is not clear what the mechanism is here, or if the two factors are directly linked.</li> </ul> <h3>How can you extract this data? (Yes, this is very dull)</h3> <p>The first problem was generating the lists of articles and their metadata. The English Wikipedia category system lets us identify “living people”, but not gender; Wikidata lets us identify gender (property P21), but not reliably “living people”. However, we can creatively use the <a href="http://petscan.wmflabs.org/">petscan</a> tool to get the intersection of a SPARQL gender query + the category. Instructing it to explicitly use Wikipedia (“enwiki” in other sources > manual list) and give output as a TSV – then waiting for about fifteen minutes – leaves you with a nice clean data dump. Thanks, <a href="http://magnusmanske.de/wordpress/">Magnus</a>!</p> <p>(It’s worth noting that you can get this data with any characteristic indexed by Wikidata, or any characteristic identifiable through the Wikipedia category schema, but you will need to run a new query for each aspect you want to analyse – the exported data just has article metadata, none of the Wikidata/category information)</p> <p>The exported files contain three things that are very useful to us: article title, pageid, and length. I normalised the files like so:</p> <div class="wp-geshi-highlight-wrap5"><div class="wp-geshi-highlight-wrap4"><div class="wp-geshi-highlight-wrap3"><div class="wp-geshi-highlight-wrap2"><div class="wp-geshi-highlight-wrap"><div class="wp-geshi-highlight"><div class="bash"><pre class="de1"><span class="kw2">grep</span> <span class="br0">[</span><span class="nu0">0</span>-<span class="nu0">9</span><span class="br0">]</span> enwiki_blp_women_from_list.tsv <span class="sy0">|</span> <span class="kw2">cut</span> <span class="re5">-f</span> <span class="nu0">2</span>,<span class="nu0">3</span>,<span class="nu0">5</span> <span class="sy0">&</span>gt; women-noheader.tsv</pre></div></div></div></div></div></div></div> <p>This drops the header line (it’s the only one with no numeric characters) and extracts only the three values we care about (and conveniently saves about 20MB).</p> <p>This gives us two of the things we want (age and size) but not deletion data. For that, we fall back on inference. Any article that is put through the AFD process gets a new subpage created at “Wikipedia:Articles for deletion/PAGENAME”. It is reasonable to infer that if an article has a corresponding AFD subpage, it’s <i>probably</i> about that specific article. This is not always true, of course – names get recycled, pages get moved – but it’s a reasonable working hypothesis and hopefully the errors are evenly distributed over time. I’ve racked my brains to see if I could anticipate a noticeable difference here by gender, as this could really complicate the results, but provisionally I think we’re okay to go with it.</p> <p>To find out if those subpages exist, we turn to <a href="https://dumps.wikimedia.org/enwiki/latest/">the enwiki dumps</a>. Specifically, we want “enwiki-latest-all-titles.gz” – which, as it suggests, is a simple file listing all page titles on the wiki. Extracted, it comes to about 1GB. From this, we can extract all the AFD subpages, as so:</p> <div class="wp-geshi-highlight-wrap5"><div class="wp-geshi-highlight-wrap4"><div class="wp-geshi-highlight-wrap3"><div class="wp-geshi-highlight-wrap2"><div class="wp-geshi-highlight-wrap"><div class="wp-geshi-highlight"><div class="bash"><pre class="de1"><span class="kw2">grep</span> <span class="st0">"Articles_for_deletion/"</span> enwiki-latest-all-titles <span class="sy0">|</span> <span class="kw2">cut</span> <span class="re5">-f</span> <span class="nu0">2</span> <span class="sy0">|</span> <span class="kw2">sort</span> <span class="sy0">|</span> <span class="kw2">uniq</span> <span class="sy0">|</span> <span class="kw2">cut</span> <span class="re5">-f</span> <span class="nu0">2</span> <span class="re5">-d</span> <span class="sy0">/</span> <span class="sy0">|</span> <span class="kw2">sort</span> <span class="sy0">|</span> <span class="kw2">uniq</span> <span class="sy0">&</span>gt; afds</pre></div></div></div></div></div></div></div> <p>This extracts all the AFD subpages, removes any duplicates (since eg talkpages are listed here as well), and sorts the list alphabetically. There are about 424,000 of them.</p> <p>Going back to our original list of articles, we want to bin them by age. To a first approximation, <tt>pageid</tt> is sequential with age – it’s assigned when the page is first created. There are some big caveats here; for example, a page being created as a redirect and later expanded will have the ID of its initial creation. Pages being deleted and recreated may get a new ID, pages which are merged may end up with either of the original IDs, and some complicated page moves may end up with the original IDs being lost. But, for the majority of pages, it’ll work out okay.</p> <p>To correlate pageID to age, I did a bit of speculative guessing to find an item created on 1 January and 1 July every year back to 2009 (eg <a href="https://en.wikipedia.org/w/index.php?curid=43190000&action=info#Edit_history">pageid 43190000</a> was created at 11am on 1 July 2014). I could then use these to extract the articles corresponding to each period as so:</p> <div class="wp-geshi-highlight-wrap5"><div class="wp-geshi-highlight-wrap4"><div class="wp-geshi-highlight-wrap3"><div class="wp-geshi-highlight-wrap2"><div class="wp-geshi-highlight-wrap"><div class="wp-geshi-highlight"><div class="bash"><pre class="de1">... <span class="kw2">awk</span> <span class="re5">-F</span> <span class="st_h">'\t'</span> <span class="st_h">'$2 &gt;= 41516000 &amp;&amp; $2 &lt; 43190000'</span> <span class="sy0">&</span>lt; men-noheader.tsv <span class="sy0">&</span>gt; bins<span class="sy0">/</span><span class="nu0">2014</span>-<span class="nu0">1</span>-M <span class="kw2">awk</span> <span class="re5">-F</span> <span class="st_h">'\t'</span> <span class="st_h">'$2 &gt;= 43190000 &amp;&amp; $2 &lt; 44909000'</span> <span class="sy0">&</span>lt; men-noheader.tsv <span class="sy0">&</span>gt; bins<span class="sy0">/</span><span class="nu0">2014</span>-<span class="nu0">2</span>-M ...</pre></div></div></div></div></div></div></div> <p>This finds all items with a pageid (in column #2 of the file) between the specified values, and copies them into the relevant bin. Run once for men and once for women.</p> <p>Then we can run a short report, along these lines (the original had loops in it):</p> <div class="wp-geshi-highlight-wrap5"><div class="wp-geshi-highlight-wrap4"><div class="wp-geshi-highlight-wrap3"><div class="wp-geshi-highlight-wrap2"><div class="wp-geshi-highlight-wrap"><div class="wp-geshi-highlight"><div class="bash"><pre class="de1"> <span class="kw2">cut</span> <span class="re5">-f</span> <span class="nu0">1</span> bins<span class="sy0">/</span><span class="nu0">2014</span>-<span class="nu0">1</span>-M <span class="sy0">|</span> <span class="kw2">sort</span> <span class="sy0">&</span>gt; temp-M <span class="kw3">echo</span> <span class="re5">-e</span> <span class="nu0">2014</span>-<span class="nu0">1</span>-M<span class="st0">"<span class="es1">\t</span>M<span class="es1">\t</span>"</span><span class="sy0">`</span><span class="kw2">cat</span> bins<span class="sy0">/</span><span class="nu0">2014</span>-<span class="nu0">1</span>-M <span class="sy0">|</span> <span class="kw2">wc</span> -l<span class="sy0">`</span><span class="st0">"<span class="es1">\t</span>"</span><span class="sy0">`</span><span class="kw2">awk</span> <span class="st_h">'{ total += $3; count++ } END { print total/count }'</span> bins<span class="sy0">/</span><span class="nu0">2014</span>-<span class="nu0">1</span>-M<span class="sy0">`</span><span class="st0">"<span class="es1">\t</span>"</span><span class="sy0">`</span><span class="kw2">comm</span> <span class="re5">-1</span> <span class="re5">-2</span> temp-M afds <span class="sy0">|</span> <span class="kw2">wc</span> -l<span class="sy0">`</span> <span class="sy0">&</span>gt;<span class="sy0">&</span>gt; report.tsv</pre></div></div></div></div></div></div></div> <p>This adds a line to the file <tt>report.tsv</tt> with (in order) the name of the bin, the number of entries in it, the mean value of the <tt>length</tt> column, and a count of the number which also match names in the <tt>afds</tt> file. (The use of the <tt>temp-M</tt> file is to deal with the fact that the comm tool needs properly sorted input).</p> <p>After that, generating the data is lovely and straightforward – drop the report into a spreadsheet and play around with it.</p> </div> <footer class="entry-footer"> <span class="posted-on"><span class="screen-reader-text">Posted on </span><a href="https://www.generalist.org.uk/blog/2019/gender-and-deletion-on-wikipedia/" rel="bookmark"><time class="entry-date published" datetime="2019-05-06T21:36:49+01:00">May 6, 2019</time><time class="updated" datetime="2022-08-14T22:07:02+01:00">August 14, 2022</time></a></span><span class="cat-links"><span class="screen-reader-text">Categories </span><a href="https://www.generalist.org.uk/blog/authors/andrew/" rel="category tag">Andrew</a></span><span class="tags-links"><span class="screen-reader-text">Tags </span><a href="https://www.generalist.org.uk/blog/tags/data/" rel="tag">data</a>, <a href="https://www.generalist.org.uk/blog/tags/statistics/" rel="tag">statistics</a>, <a href="https://www.generalist.org.uk/blog/tags/wikidata/" rel="tag">wikidata</a>, <a href="https://www.generalist.org.uk/blog/tags/wikipedia/" rel="tag">wikipedia</a></span><span class="comments-link"><a href="https://www.generalist.org.uk/blog/2019/gender-and-deletion-on-wikipedia/#comments">3 Comments<span class="screen-reader-text"> on Gender and deletion on Wikipedia</span></a></span> </footer> </article> <article id="post-1469" class="post-1469 post type-post status-publish format-standard hentry category-andrew tag-data tag-politics"> <header class="entry-header"> <h2 class="entry-title"><a href="https://www.generalist.org.uk/blog/2017/on-article-50/" rel="bookmark">On Article 50</a></h2> </header> <div class="entry-content"> <p>Since early last year, I live in Islington North (pretty much bang in the middle of it, in fact). This means that my MP is one Jeremy Corbyn.</p> <p>This is a bit odd. I’ve never been represented by someone who wasn’t a backbencher before (at least, not at Westminster; for a brief while years ago my MSP was the subduedly-titled Deputy Minister for Justice). It also means that there is very little reason for me to ever <i>write</i> to my MP – his positions on something are usually front-page news and for any given topic I can figure out pretty quickly that either he’s already made a statement supporting it or disagrees with me entirely.</p> <p>But, the Article 50 vote looms, and I felt I ought to do it for once. I know he disagrees with me; I know he’s <i>whipped his party</i> that way. The letter is a cry in the dark. But, well, you do what you must do.</p> <blockquote><p>Dear Mr. Corbyn,</p> <p>I am writing in regard to the Article 50 second reading vote scheduled for Wednesday February 1st. As a constituent, I urge you to reconsider your position on this bill, and to vote against it at the second reading. </p> <p>Firstly, I wish to remind you that around 75% of your constituents voted Remain, on a turnout of 70%. Not only was Islington one of the most strongly pro-EU areas of the country, this was a larger share of the electorate than you yourself have ever received from the constituency – and it has always been a solidly Labour seat. This is a remarkable result, and I feel it is only proper that you acknowledge your constituents’ clearly expressed position here.</p> <p>Secondly, on pragmatic grounds, this bill is likely to pass without significant amendments, and thus without any controls on Brexit barring those imposed by a weak Prime Minister. As such, it is essentially handing a blank cheque to the hard right of the Conservative Party, giving them the carte blanche to engineer a Brexit most suited to their desired outcomes – xenophobic, intolerant, and devastating to the British people. This is a horrendous prospect.</p> <p>Rejecting this bill at second reading will not stop Brexit and will not invalidate the referendum. However, rejecting the bill will have a decent chance of forcing these discussions to be open, to take place on a cross-party basis, and ensure that what emerges has a chance of being positive for us all.</p> <p>Thirdly, the wider context. Internationally, the world has changed dramatically since last summer. Europe, with all its flaws, is a beacon of light and sanity compared to the United States, our closest non-EU ally. As you yourself noted yesterday, the Prime Minister’s support for Donald Trump places her firmly on the wrong side of history.</p> <p>And in this light, the referendum result has some resonance. You were one of 84 Labour members to defy a whip and vote against the invasion of Iraq. A poll conducted the same day found about 50-55% of the country in favour of the war – the same number that voted to leave the EU.</p> <p>A slim majority of the country – and the government – got it wrong a decade ago. We are not infallible. Sometimes, we all take the wrong steps and put ourselves on the wrong side of history. Now is a chance to put the brakes on and decide quite what we are doing, to move slowly and contemplatively, before continuing further.</p> <p>I urge you to vote against this bill.</p></blockquote> </div> <footer class="entry-footer"> <span class="posted-on"><span class="screen-reader-text">Posted on </span><a href="https://www.generalist.org.uk/blog/2017/on-article-50/" rel="bookmark"><time class="entry-date published" datetime="2017-01-31T23:49:16+00:00">January 31, 2017</time><time class="updated" datetime="2022-08-14T22:09:21+01:00">August 14, 2022</time></a></span><span class="cat-links"><span class="screen-reader-text">Categories </span><a href="https://www.generalist.org.uk/blog/authors/andrew/" rel="category tag">Andrew</a></span><span class="tags-links"><span class="screen-reader-text">Tags </span><a href="https://www.generalist.org.uk/blog/tags/data/" rel="tag">data</a>, <a href="https://www.generalist.org.uk/blog/tags/politics/" rel="tag">politics</a></span><span class="comments-link"><a href="https://www.generalist.org.uk/blog/2017/on-article-50/#comments">2 Comments<span class="screen-reader-text"> on On Article 50</span></a></span> </footer> </article> <article id="post-1293" class="post-1293 post type-post status-publish format-standard hentry category-andrew tag-data tag-gis tag-maps tag-wikimedia tag-wikipedia"> <header class="entry-header"> <h2 class="entry-title"><a href="https://www.generalist.org.uk/blog/2015/taking-pictures-with-flying-government-lasers/" rel="bookmark">Taking pictures with flying government lasers</a></h2> </header> <div class="entry-content"> <p>Well, sort of.</p> <p>A few weeks ago, the Environment Agency released <a href="https://environmentagency.blog.gov.uk/2015/09/18/laser-surveys-light-up-open-data/">the first tranche of their LIDAR survey data</a>. This covers (most of) England, at varying resolution from 2m to 25cm, made via <a href="https://en.wikipedia.org/wiki/Lidar">LIDAR airborne survey</a>.</p> <p>It’s great fun. After a bit of back-and-forth (and hastily figuring out how to use QGIS), here’s two rendered images I made of Durham, one with buildings and one without, now on Commons:</p> <p><a href="https://commons.wikimedia.org/wiki/File:LIDAR image of the City of Durham, 1m resolution, with buildings.tiff"><img decoding="async" src="https://upload.wikimedia.org/wikipedia/commons/thumb/0/07/LIDAR_image_of_the_City_of_Durham%2C_1m_resolution%2C_with_buildings.tiff/lossy-page1-640px-LIDAR_image_of_the_City_of_Durham%2C_1m_resolution%2C_with_buildings.tiff.jpg"></a></p> <p><a href="https://commons.wikimedia.org/wiki/File:LIDAR image of the City of Durham, 1m resolution.tiff"><img decoding="async" src="https://upload.wikimedia.org/wikipedia/commons/thumb/a/a4/LIDAR_image_of_the_City_of_Durham%2C_1m_resolution.tiff/lossy-page1-640px-LIDAR_image_of_the_City_of_Durham%2C_1m_resolution.tiff.jpg"></a></p> <p>The first is shown with buildings, the second without. Both are at 1m resolution, the best currently available for the area. Note in particular the very striking embankment and cutting for the railway viaduct (top left). These look like they could be very useful things to produce for Commons, especially since it’s – effectively – very recent, openly licensed, aerial imagery…</p> <h4>1. Selecting a suitable area</h4> <p>Generating these was, on the whole, fairly easy. First, install <a href="http://www.qgis.org/en/site/">QGIS</a> (simplicity itself on a linux machine, probably not too much hassle elsewhere). Then, go to the <a href="http://environment.data.gov.uk/ds/survey#/">main data page</a> and find the area you’re interested in. It’s arranged on an Ordnance Survey grid – click anywhere on the map to select a grid square. Major grid squares (Durham is NZ24) are 10km by 10km, and all data will be downloaded in a zip file containing tiles for that particular region.</p> <p>Let’s say we want to try Cambridge. The TL45 square neatly cuts off North Cambridge but most of the city is there. If we look at the bottom part of the screen, it offers “Digital Terrain Model” at 2m and 1m resolution, and “Digital Surface Model” likewise. The DTM is the version just showing the terrain (no buildings, trees, etc) while the DSM has all the surface features included. Let’s try the DSM, as Cambridge is not exactly mountainous. The “on/off” slider will show exactly what the DSM covers in this area, though in Cambridge it’s more or less “everything”.</p> <p>While this is downloading, let’s pick our target area. Zooming in a little further will show thinner blue lines and occasional superimposed blue digits; these define the smaller squares, 1 km by 1 km. For those who don’t remember learning to read OS maps, the number on the left and the number on the bottom, taken together, define the square. So the sector containing all the colleges along the river (a dense clump of black-outlined buildings) is TL4458.</p> <h4>2. Rendering a single tile</h4> <p>Now your zip file has downloaded, drop all the files into a directory somewhere. Note that they’re all named something like <tt>tl4356_DSM_1m.asc</tt>. Unsurprisingly, this means the 1m DSM data for square TL4356.</p> <p>Fire up QGIS, go to <tt>Layer > Add raster layer</tt>, and select your tile – in this case, TL4458. You’ll get a crude-looking monochrome image, immediately recognisable by a broken white line running down the middle. This is the Cam. If you’re seeing this, great, everything’s working so far. (This step is very helpful to check you are looking at the right area)</p> <p>Now, let’s make the image. <tt>Project > New</tt> to blank everything (no need to save). Then <tt>Raster > Analysis > DEM (terrain models)</tt>. In the first box, select your chosen input file. In the next box, the output filename – with a .tif suffix. (Caution, linux users: make sure to enter or select a path here, otherwise it seems to default to home). Leave everything else as default – all unticked and mode: hillshade. Click OK, and a few seconds later it’ll give a completed message; cancel out of the dialogue box at this point. It’ll be displaying something like this:</p> <p><a href="https://commons.wikimedia.org/wiki/File:LIDAR_image_of_an_area_of_Cambridge,_1m_resolution,_with_buildings.tiff"><img decoding="async" src="https://upload.wikimedia.org/wikipedia/commons/thumb/1/1a/LIDAR_image_of_an_area_of_Cambridge%2C_1m_resolution%2C_with_buildings.tiff/lossy-page1-480px-LIDAR_image_of_an_area_of_Cambridge%2C_1m_resolution%2C_with_buildings.tiff.jpg"></a></p> <p>Congratulations! Your first LIDAR rendering. You can quit out of QGIS (you can close without saving, your converted file is saved already) and open this up as a normal TIFF file now; it’ll be about 1MB and cover an area 1km by 1km. If you look closely, you can see some surprisingly subtle details despite the low resolution – the low walls outside Kings College, for example, or cars on the Queen’s Road – Madingley Road roundabout by the top left.</p> <h4>3. Rendering several tiles</h4> <p>Rendering multiple squares is a little trickier. Let’s try doing Barton, which conveniently fits into two squares – TL4055 and TL4155. Open QGIS up, and render TL4055 as above, through <tt>Raster > Analysis > DEM (terrain models)</tt>. Then, with the dialogue window still open, select TL4155 (and a new output filename) and run it again. Do this for as many files as you need.</p> <p>After all the tiles are prepared, clear the screen by starting a new project (again, no need to save) and go to <tt>Raster > Miscellaneous > Merge</tt>. In “Input files”, select the two exports you’ve just done. In “Output file”, pick a suitable filename (again ending in .tif). Hit OK, let it process, then close the dialog. You can again close QGIS without saving, as the export’s complete.</p> <p>The rendering system embeds coordinates in the files, which means that when they’re assembled and merged they’ll automatically slot together in the correct position and orientation – no need to manually tile them. The result should look like this:</p> <p><a href="https://commons.wikimedia.org/wiki/File:LIDAR_image_of_Barton,_Cambridgeshire,_1m_resolution,_with_buildings.tiff"><img decoding="async" src="https://upload.wikimedia.org/wikipedia/commons/thumb/d/da/LIDAR_image_of_Barton%2C_Cambridgeshire%2C_1m_resolution%2C_with_buildings.tiff/lossy-page1-640px-LIDAR_image_of_Barton%2C_Cambridgeshire%2C_1m_resolution%2C_with_buildings.tiff.jpg"></a></p> <p>The odd black bit in the top right is the edge of the flight track – there’s not quite comprehensive coverage. This is a mainly agricultural area, and you can see field markings – some quite detailed, and a few bits on the bottom of the right-hand tile that might be traces of old buildings.</p> <p>So… go forth! Make LIDAR images! See what you can spot…</p> <h4>4. Command-line rendering in bulk</h4> <p>Richard Symonds (who started me down this rabbit-hole) points out <a href="http://chris-osm.blogspot.co.uk/2015/09/more-lidar-goodness.html">this</a> very useful post, which explains how to do the rendering and merging via the command line. Let’s try the entire Durham area; 88 files in NZ24, all dumped into a single directory –</p> <blockquote><p> <tt>for i in `ls *.asc` ; do gdaldem hillshade -compute_edges $i $i.tif ; done</tt></p> <p><tt>gdal_merge.py -o NZ24-area.tif *.tif</tt></p> <p><tt><br /> </tt><tt></tt><tt>rm *.asc.tif</tt></p></blockquote> <p>In order, that a) runs the hillshade program on each individual source file ; b) assembles them into a single giant image file; c) removes the intermediate images (optional, but may as well tidy up). The <tt>-compute_edges</tt> flag helpfully removes the thin black lines between sectors – I should have turned it on in the earlier sections!</p> </div> <footer class="entry-footer"> <span class="posted-on"><span class="screen-reader-text">Posted on </span><a href="https://www.generalist.org.uk/blog/2015/taking-pictures-with-flying-government-lasers/" rel="bookmark"><time class="entry-date published" datetime="2015-10-02T21:38:00+01:00">October 2, 2015</time><time class="updated" datetime="2019-09-17T19:20:14+01:00">September 17, 2019</time></a></span><span class="cat-links"><span class="screen-reader-text">Categories </span><a href="https://www.generalist.org.uk/blog/authors/andrew/" rel="category tag">Andrew</a></span><span class="tags-links"><span class="screen-reader-text">Tags </span><a href="https://www.generalist.org.uk/blog/tags/data/" rel="tag">data</a>, <a href="https://www.generalist.org.uk/blog/tags/gis/" rel="tag">gis</a>, <a href="https://www.generalist.org.uk/blog/tags/maps/" rel="tag">maps</a>, <a href="https://www.generalist.org.uk/blog/tags/wikimedia/" rel="tag">wikimedia</a>, <a href="https://www.generalist.org.uk/blog/tags/wikipedia/" rel="tag">wikipedia</a></span><span class="comments-link"><a href="https://www.generalist.org.uk/blog/2015/taking-pictures-with-flying-government-lasers/#comments">5 Comments<span class="screen-reader-text"> on Taking pictures with flying government lasers</span></a></span> </footer> </article> <article id="post-1232" class="post-1232 post type-post status-publish format-standard hentry category-andrew tag-canada tag-data tag-history tag-statistics tag-ww1"> <header class="entry-header"> <h2 class="entry-title"><a href="https://www.generalist.org.uk/blog/2015/canadian-self-reported-birthday-data/" rel="bookmark">Canadian self-reported birthday data</a></h2> </header> <div class="entry-content"> <p>In the <a href="https://www.generalist.org.uk/blog/2015/when-do-you-think-you-were-born/">last post</a>, we saw strong evidence for a “memorable date” bias in self-reported birthday information among British men born in the late 19th century. In short, they were disproportionately likely to <i>think</i> they were born on an “important day” such as Christmas.</p> <p>It would be great to compare it to other sources. However, finding a suitable dataset is challenging. We need a sample covering a large number of men, over several years, and which is unlikely to be cross-checked or drawn from official documentation such as birth certificates or parish registers. It has to explicitly list full birthdates (not just month or year)</p> <p>WWI enlistment datasets are quite promising in this regard – lots of men, born about the same time, turning up and stating their details without particularly much of a reason to bias individual dates. The main British records have (famously) long since burned, but the Australian and Canadian records survive. Unfortunately, the <a href="https://www.aif.adfa.edu.au/search">Australian index</a> does not include dates of birth, but the <a href="http://www.bac-lac.gc.ca/eng/discover/military-heritage/first-world-war/first-world-war-1914-1918-cef/Pages/search.aspx">Canadian index</a> does (at least, when known). So, does it tell us anything?</p> <p>The index is available as <a href="http://data.gc.ca/data/en/dataset/6a6e6a79-9e2a-48cc-99ec-163da26d15e9">a 770mb+ XML blob</a> (oh, dear). Running this through <tt>xmllint</tt> produces a nicely formatted file with approximately 575,000 birthdays for 622,000 entries. It’s formatted in such a way as to imply there may be multiple birthdates listed for a single individual (presumably if there’s contradictory data?), but I couldn’t spot any cases. There’s also about ten thousand who don’t have nicely formatted dd/mm/yyyy entries; let’s omit those for now. Quick and dirt but probably representative.</p> <p>And so…</p> <p><img decoding="async" src="https://www.generalist.org.uk/blog/wp-content/uploads/2015/02/canadiandata.png" width=500px></p> <p>There’s clearly a bit more seasonality here than in the British data (up in spring, down in winter), but also the same sort of unexpected one-day spikes and troughs. As this is quite rough, I haven’t corrected for seasonality, but we still see something interesting.</p> <p>The highest ten days are: 25 December (1.96), 1 January (1.77), 17 March (1.56), 24 May (1.52), 1 May (1.38), 15 August (1.38), 12 July (1.36), 15 September (1.34), 15 March (1.3).</p> <p>The lowest ten days are: 30 December (0.64), 30 January (0.74), 30 October (0.74), 30 July (0.75), 30 May (0.78), 13 November (0.78), 30 August (0.79), 26 November (0.80), 30 March (0.81), 12 December (0.81).</p> <p>The same strong pattern for “memorable days” that we saw with the UK is visible in the top ten – Christmas, New Year, St. Patrick’s, Victoria Day, May Day, [nothing], 12 July, [nothing], [nothing].</p> <p>Two of these are distinctively “Canadian” – both 24 May (the Queen’s birthday/Victoria Day) and 12 July (the Orange Order marches) are above average in the British data, but not as dramatically as they are here. Both appear to have been relatively more prominent in late-19th/early-20th century Canada than in the UK. Canada Day/Dominion Day (1 July) is above average but does not show up as sharply, possibly because it does not appear to have been widely celebrated until after WWI.</p> <p>One new pattern is the appearance of the 15th of the month in the top 10. This was suggested as likely in the <a href="http://www.panix.com/~murphy/bday.html">US life insurance analysis</a> and I’m interested to see it showing up here. Another oddity is leap years – in the British data, 29 February was dramatically undercounted. In the Canadian data, it’s strongly overcounted – just not quite enough to get into the top ten. 28 February (1.28), 29 February (1.27) and 1 March (1.29) are all “memorable”. I don’t have an explanation for this but it does suggest an interesting story.</p> <p>Looking at the lowest days, we see the same pattern of 30/xx dates being very badly represented – seven of the ten lowest dates are 30th of the month…. and all from days where there were 31 days in the month. This is exactly the same pattern we observed in UK data, and I just don’t have any convincing reason to guess why. The other three dates all fall in low-birthrate months, </p> <p>So, in conclusion:</p> <ul> <li>Both UK and Canadian data from WWI show a strong bias for people to self-report their birthday as a “memorable day”; <li>“Memorable” days are commonly a known and fixed festival, such as Christmas; <li>Overreporting of arbitrary numbers like the 15th of the month are more common in Canada (& possibly the US?) than the UK; <li>The UK and Canadian samples seem to treat 29 February very differently – Canadians overreport, British people underreport; <li>There is a strong bias against reporting the 30th of the month <i>particularly in months with 31 days</i> </ul> <p>Thoughts (or additional data sources) welcome.</p> </div> <footer class="entry-footer"> <span class="posted-on"><span class="screen-reader-text">Posted on </span><a href="https://www.generalist.org.uk/blog/2015/canadian-self-reported-birthday-data/" rel="bookmark"><time class="entry-date published" datetime="2015-02-22T16:17:28+00:00">February 22, 2015</time><time class="updated" datetime="2015-02-22T16:21:02+00:00">February 22, 2015</time></a></span><span class="cat-links"><span class="screen-reader-text">Categories </span><a href="https://www.generalist.org.uk/blog/authors/andrew/" rel="category tag">Andrew</a></span><span class="tags-links"><span class="screen-reader-text">Tags </span><a href="https://www.generalist.org.uk/blog/tags/canada/" rel="tag">canada</a>, <a href="https://www.generalist.org.uk/blog/tags/data/" rel="tag">data</a>, <a href="https://www.generalist.org.uk/blog/tags/history/" rel="tag">history</a>, <a href="https://www.generalist.org.uk/blog/tags/statistics/" rel="tag">statistics</a>, <a href="https://www.generalist.org.uk/blog/tags/ww1/" rel="tag">ww1</a></span><span class="comments-link"><a href="https://www.generalist.org.uk/blog/2015/canadian-self-reported-birthday-data/#comments">1 Comment<span class="screen-reader-text"> on Canadian self-reported birthday data</span></a></span> </footer> </article> <article id="post-1220" class="post-1220 post type-post status-publish format-standard hentry category-andrew tag-data tag-history tag-statistics tag-ww1"> <header class="entry-header"> <h2 class="entry-title"><a href="https://www.generalist.org.uk/blog/2015/when-do-you-think-you-were-born/" rel="bookmark">When do you think you were born?</a></h2> </header> <div class="entry-content"> <p>Back in <a href="https://www.generalist.org.uk/blog/2015/back-to-the-army-again/">the last post</a>, we were looking at a sample of dates-of-birth in post-WWI Army records.</p> <p>(To recap – this is a dataset covering every man who served in the British Army after 1921 and who had a date of birth in or before 1900. 371,716 records in total, from 1864 to 1900, strongly skewed towards the recent end.)</p> <p>I’d suggested that there was an “echo” of 1914/15 false enlistment in there, but after a bit of work I’ve not been able to see it. However, it did throw up some other very interesting things. Here’s the graph of birthdays.</p> <p><img decoding="async" src="https://www.generalist.org.uk/blog/wp-content/uploads/2015/02/armybirths-raw.png" width=500px></p> <p>Two things immediately jump out. The first is that the graph, very gently, slopes upwards. The second is that there are some wild outliers.</p> <p>The first one is quite simple to explain; this data is not a sample of men born in a given year, but rather those in the army a few decades later. The graph in the previous post shows a very strong skew towards younger ages, so for <i>any given year</i> we’d expect to find marginally more December births than January ones. I’ve normalised the data to reflect this – calculated what the expected value for any given day would be assuming a linear increase, then calculated the ratio of reported to expected births. [For 29 February, I quartered its expected value]</p> <p><img decoding="async" src="https://www.generalist.org.uk/blog/wp-content/uploads/2015/02/armybirths-normalised.png" width=500px></p> <p>There are hints at a seasonal pattern here, but not a very obvious one. January, February, October and November are below average, March and September above average, and the rest of the spring-summer is hard to pin down. (For quite an interesting discussion on “European” and “American” birth seasonality, see <a href="http://www.canpopsoc.ca/CanPopSoc/assets/File/publications/journal/CSPv20n1p1.pdf">this Canadian paper</a>)</p> <p>The interesting bit is the outliers, which are apparent in both graphs. </p> <p>The most overrepresented days are, in order of frequency, 1 January (1.8), 25 December (1.43), 17 March (1.33), 28 February (1.27), 14 February (1.22), 1 May (1.22), 11 November (1.19), 12 August (1.17), 2 February (1.15), and 10 October (1.15). Conversely, the most underrepresented days are 29 February (0.67 after adjustment), 30 July (0.75), 30 August (0.78), 30 January (0.81), 30 March (0.82), and 30 May (0.84).</p> <p>Of the ten most common days, seven are significant festivals. In order: New Year’s Day, Christmas Day, St. Patrick’s Day, [nothing], Valentine’s Day, May Day, Martinmas, [nothing], Candlemas, [nothing].</p> <p>Remember, the underlying bias of most data is that it tells you what people put into the system, not what really happened. So, what we have is a dataset of what a large sample of men born in late nineteenth century Britain <i>thought</i> their birthdays were, or of the way they pinned them down when asked by an official. “Born about Christmastime” easily becomes “born 25 December” when it has to go down on a form. (Another frequent artefact is <a href="http://www.panix.com/~murphy/bday.html">overrepresentation of 1-xx or 15-xx dates</a>, but I haven’t yet looked for this.) People were substantially more likely to remember a birthday as associated with a particular festival or event than they were to remember a random date.</p> <p>It’s not all down to being memorable, of course; 1 January is probably in part a data recording artefact. I strongly suspect that at some point in the life of these records, someone’s said “record an unknown date as 1/1/xx”. </p> <p>The lowest days are strange, though. 29 February is easily explained – even correcting for it being one quarter as common as other days, many people would probably put 28 February or 1 March on forms for simplicity. (This also explains some of the 28 February popularity above). But all of the other five are 30th of the month – and all are 30th of a 31-day month. I have no idea what might explain this. I would really, really love to hear suggestions. </p> <p>One last, and possibly related, point – each month appears to have its own pattern. The first days of the month are overrepresented; the last days underrepresented. (The exception is December and possibly September). This is visible in both normalised and raw data, and I’m completely lost as to what might cause it…</p> </div> <footer class="entry-footer"> <span class="posted-on"><span class="screen-reader-text">Posted on </span><a href="https://www.generalist.org.uk/blog/2015/when-do-you-think-you-were-born/" rel="bookmark"><time class="entry-date published" datetime="2015-02-16T23:12:55+00:00">February 16, 2015</time><time class="updated" datetime="2015-02-17T09:17:38+00:00">February 17, 2015</time></a></span><span class="cat-links"><span class="screen-reader-text">Categories </span><a href="https://www.generalist.org.uk/blog/authors/andrew/" rel="category tag">Andrew</a></span><span class="tags-links"><span class="screen-reader-text">Tags </span><a href="https://www.generalist.org.uk/blog/tags/data/" rel="tag">data</a>, <a href="https://www.generalist.org.uk/blog/tags/history/" rel="tag">history</a>, <a href="https://www.generalist.org.uk/blog/tags/statistics/" rel="tag">statistics</a>, <a href="https://www.generalist.org.uk/blog/tags/ww1/" rel="tag">ww1</a></span><span class="comments-link"><a href="https://www.generalist.org.uk/blog/2015/when-do-you-think-you-were-born/#comments">3 Comments<span class="screen-reader-text"> on When do you think you were born?</span></a></span> </footer> </article> <article id="post-1204" class="post-1204 post type-post status-publish format-standard hentry category-andrew tag-data tag-history tag-statistics tag-ww1"> <header class="entry-header"> <h2 class="entry-title"><a href="https://www.generalist.org.uk/blog/2015/back-to-the-army-again/" rel="bookmark">Back to the Army again</a></h2> </header> <div class="entry-content"> <p>In the winter of 1918-19, the British government found itself in something of a quandary. On the one hand, hurrah, the war was over! Everyone who had signed up to serve for “three years or the duration” could go home. And, goodness, did they want to go home.</p> <p>On the other hand, the war… well it wasn’t really over. There were British troops fighting deep inside Russia; there were large garrisons sitting in western Germany (and other, less probable, places) in case the peace talks collapsed; there was unrest around the Empire and fears about Bolsheviks at home.</p> <p>So they <a href="http://www.1914-1918.net/reenlistment.html">raised another army</a>. Anyone in the army who volunteered to re-enlist got a cash payment of £20 to £50 (no small sum in 1919); two month’s leave with full pay; plus comparable pay to that in wartime and a separation allowance if he was married. Demobilisation continued for everyone else (albeit slowly), and by 1921, this meant that everyone in the Army was either a very long-serving veteran, a new volunteer who’d not been conscripted during wartime (so born 1901 onwards) or – I suspect the majority – re-enlisted men working through their few years service.</p> <p>For administrative convenience, all records of men who left up to 1921 were set aside and stored by a specific department; the “live” records, including more or less everyone who reenlisted, continued with the War Office. They were never transferred – and, unlike the pre-1921 records, they were not lost in a bombing raid in 1940.</p> <p>The MoD has just released an interesting dataset <a href="http://1914-1918.invisionzone.com/forums/index.php?showtopic=221706&hl=">following an FOI request</a> – it’s <a href="https://www.gov.uk/government/publications/foi-responses-released-by-mod-week-commencing-1-december-2014">an index of these “live” service records</a>. The records cover all men in the post-1921 records with a DoB prior to 1901, and thus <i>almost</i> everyone in it would have either remained in service or re-enlisted – there would be a small proportion of men born in 1900 who escaped conscription (roughly 13% of them would have turned 18 just after 11/11/18), and a small handful of men will have re-enlisted or transferred in much later, but otherwise – they all would have served in WWI and chosen to remain or to return very soon after being released. </p> <p>So, what does this tell us? Well, for one thing, there’s almost 317,000 of them. 4,864 were called Smith, 3,328 Jones, 2,104 Brown, 1,172 Black, etc. 12,085 were some form of Mac or Mc. And there are eight Singhs, which looks like an interesting story to trace about early immigrants.</p> <p>But, you know, data cries out to be graphed. So here’s the dates of birth.</p> <p><img decoding="async" src="https://www.generalist.org.uk/blog/wp-content/uploads/2015/01/1919-servicemen.png" width=500px></p> <p>Since the 1900 births are probably an overcount for reenlistments, I’ve left these off.</p> <p>It’s more or less what you’d expect, but on close examination a little story emerges. Look at 1889/90; there’s a real discontinuity here. Why would this be?</p> <p>Pre-war army enlistments were not for ‘the duration’ (there was nothing to have a duration of!) but for seven years service and five in the reserves. There was a rider on this – if war broke out, you wouldn’t be discharged until the crisis was over. The men born 1900 would have enlisted in 1908 and been due for release to the reserves in 1915. Of course, that never happened… and so, in 1919, many of these men would have been 29, knowing no other career than soldiering. Many would have been thrilled to get out – and quite a few more would have considered it, and realised they had no trade, and no great chance of good employment. As Kipling <a href="http://www.kiplingsociety.co.uk/poems_back.htm">had it in 1894</a>:</p> <blockquote><p><i>A man o’ four-an’-twenty what ‘asn’t learned of a trade—<br /> Except “Reserve” agin’ him—’e’d better be never made.</i></p></blockquote> <p>It probably wasn’t much better for him in 1919.</p> <p>Moving right a bit, 1896-97 also looks odd – this is the only point in the data where it goes <i>backwards</i>, with marginally more men born in 1896 than 1897. What happened here?</p> <p>Anyone born before August 1896 was able to rush off and enlist at the start of the war; anyone born after that date would either have to wait, or lie. Does this reflect a distant echo of people giving false ages in 1914/15 and still having them on the paperwork at reenlistment? More research no doubt needed, but it’s an interesting thought.</p> </div> <footer class="entry-footer"> <span class="posted-on"><span class="screen-reader-text">Posted on </span><a href="https://www.generalist.org.uk/blog/2015/back-to-the-army-again/" rel="bookmark"><time class="entry-date published" datetime="2015-01-24T14:35:05+00:00">January 24, 2015</time><time class="updated" datetime="2015-02-16T22:09:19+00:00">February 16, 2015</time></a></span><span class="cat-links"><span class="screen-reader-text">Categories </span><a href="https://www.generalist.org.uk/blog/authors/andrew/" rel="category tag">Andrew</a></span><span class="tags-links"><span class="screen-reader-text">Tags </span><a href="https://www.generalist.org.uk/blog/tags/data/" rel="tag">data</a>, <a href="https://www.generalist.org.uk/blog/tags/history/" rel="tag">history</a>, <a href="https://www.generalist.org.uk/blog/tags/statistics/" rel="tag">statistics</a>, <a href="https://www.generalist.org.uk/blog/tags/ww1/" rel="tag">ww1</a></span><span class="comments-link"><a href="https://www.generalist.org.uk/blog/2015/back-to-the-army-again/#comments">1 Comment<span class="screen-reader-text"> on Back to the Army again</span></a></span> </footer> </article> </main> </section> </div> <footer id="colophon" class="site-footer"> <div class="site-info"> <a href="https://wordpress.org/" class="imprint"> Proudly powered by WordPress </a> </div> </footer> </div> <script src="https://www.generalist.org.uk/blog/wp-content/themes/twentyfifteen/js/skip-link-focus-fix.js?ver=20141028" id="twentyfifteen-skip-link-focus-fix-js"></script> <script id="twentyfifteen-script-js-extra"> var screenReaderText = {"expand":"<span class=\"screen-reader-text\">expand child menu<\/span>","collapse":"<span class=\"screen-reader-text\">collapse child menu<\/span>"}; </script> <script src="https://www.generalist.org.uk/blog/wp-content/themes/twentyfifteen/js/functions.js?ver=20221101" id="twentyfifteen-script-js"></script> </body> </html>