ConferenceCorpus/statistics

Introduction

SQL Query for Event Series completion

SELECT 
   series,
   min(ordinal) as minOrdinal, 
   max(ordinal) as maxOrdinal,
   avg(ordinal) as avgOrdinal,
   max(Ordinal)-min(Ordinal) as available,
   (max(Ordinal)-min(Ordinal)) /(max(Ordinal)-1.0) as completeness
FROM event_dblp
Where ordinal is not null 
group by series
order by 6 desc

Event Signature Availability

see also http://conferencecorpus.bitplan.com/query/SignatureAvailability

signature completeness combined

signature completeness of acronym

signature completeness of startDate

signature completeness of ordinal

signature completeness of year

signature completeness of title

signature completeness of city

signature completeness of country

[[File:completeSignature_country.png|600px]

Ordinal histogramms

The Zipf digramms show the log frequency (leaving out the first event, since typically the decline begins from the second event)

confref

To few available ordinals for analysis.

CEUR-WS

sql query

SELECT ordinal
    FROM event_ceurws
    where ordinal is not null
    and ordinal < 50

ceurws ordinals

confref

not enough data

Crossref

sql query

SELECT ordinal
    FROM event_crossref
    where ordinal is not null
    and ordinal < 50

crossref ordinals

dblp

sql query

SELECT ordinal
    FROM event_dblp
    where ordinal is not null
    and ordinal < 50

dblp ordinals

GND

sql query

SELECT ordinal
    FROM event_gnd
    where ordinal is not null
    and ordinal < 50

gnd ordinals

OpenResearch

sql query

SELECT ordinal
    FROM event_or
    where ordinal is not null
    and ordinal < 50

or ordinals

OpenResearch

sql query

SELECT ordinal
    FROM event_orbackup
    where ordinal is not null
    and ordinal < 50

orbackup ordinals

OpenResearch

sql query

SELECT ordinal
    FROM event_orclone
    where ordinal is not null
    and ordinal < 50

orclone ordinals

OpenResearch

sql query

SELECT ordinal
    FROM event_orclonebackup
    where ordinal is not null
    and ordinal < 50

orclonebackup ordinals

TIBKAT

sql query

SELECT ordinal
    FROM event_tibkat
    where ordinal is not null
    and ordinal < 50

tibkat ordinals

WikiCFP

sql query

SELECT ordinal
    FROM event_wikicfp
    where ordinal is not null
    and ordinal < 50

wikicfp ordinals

Wikidata

sql query

SELECT ordinal
    FROM event_wikidata
    where ordinal is not null
    and ordinal < 50

wikidata ordinals

Eventseries completeness

The queries here only give a rough estimate for an upper bound since the assumption here is that all events between max(ordinal) and min(ordinal) are available which is often not the case.

dblp

sql query

SELECT 
       series,
       min(ordinal) as minOrdinal, 
       max(ordinal) as maxOrdinal,
       avg(ordinal) as avgOrdinal,
       max(Ordinal)-min(Ordinal) as ordinalRange,
       (max(Ordinal)-min(Ordinal)) /(max(Ordinal)-1.0) as completeness
FROM event_dblp
WHERE ordinal is not null 
GROUP BY series
ORDER by 6 DESC

event series completeness of dblp

OpenResearch

sql query

SELECT 
       inEventSeries,
       min(ordinal) as minOrdinal, 
       max(ordinal) as maxOrdinal,
       avg(ordinal) as avgOrdinal,
       max(Ordinal)-min(Ordinal) as ordinalRange,
       (max(Ordinal)-min(Ordinal)) /(max(Ordinal)-1.0) as completeness
FROM event_orclone
WHERE ordinal is not null 
GROUP BY inEventSeries
ORDER by 6 DESC

event series completeness of orclone

WikiCFP

sql query

SELECT 
       seriesId,
       min(ordinal) as minOrdinal, 
       max(ordinal) as maxOrdinal,
       avg(ordinal) as avgOrdinal,
       max(Ordinal)-min(Ordinal) as ordinalRange,
       (max(Ordinal)-min(Ordinal)) /(max(Ordinal)-1.0) as completeness
FROM event_wikicfp
WHERE ordinal is not null 
GROUP BY seriesId
ORDER by 6 DESC

event series completeness of wikicfp

Wikidata

sql query

SELECT 
       eventInSeriesId,
       min(ordinal) as minOrdinal, 
       max(ordinal) as maxOrdinal,
       avg(ordinal) as avgOrdinal,
       max(Ordinal)-min(Ordinal) as ordinalRange,
       (max(Ordinal)-min(Ordinal)) /(max(Ordinal)-1.0) as completeness
FROM event_wikidata
WHERE ordinal is not null 
GROUP BY eventInSeriesId
ORDER by 6 DESC

event series completeness of wikidata

Eventseries completeness by acronym

Not all data sources have an eventseries reference per event. As as alternative approach we analyzed the acronyms to group event series. Here we also calculated the number of distinct ordinals.

The results will in some cases mix data from event series with ambiguous acronyms and therefore give a more positive impression again. So again this is an estimate for an upper bound of the event series completion.

Python Code for Eventseries completeness analyis =

   def testSeriesCompletenessHistogrammByAcronym(self):
        '''
        acronym based histogramms
        '''
        def histogrammSettings(plot):
            '''
            optional callback to add more data to histogramm
            '''
            pass
        
        debug = False
        self.figureList=FigureList(caption="event Series completeness by acronym",figureListLabel="eventcompa",cols=3)
        for dataSource in DataSource.sources.values():
            if dataSource.name in ["acm","confref"]:
                continue
            print(dataSource)
            histOutputFileName=f"eventSeriesCompletionByAcronymHistogramm_{dataSource.name}.png"
            sqlQuery = """SELECT acronym, ordinal
                FROM %s
                """ % (dataSource.tableName)
            sqlDB = EventStorage.getSqlDB()
            lod = sqlDB.query(sqlQuery)
            series = {}
            acronymRegexp = r'(?P<acronym>[A-Z]+)\s*[0-9]+'
            for d in lod:
                acronym = d.get('acronym')
                if acronym:
                    match = re.fullmatch(acronymRegexp, acronym)
                    if match is None:
                        continue
                    seriesAcronym = match.group("acronym")
                    if isinstance(seriesAcronym, str):
                        if seriesAcronym in series:
                            series[seriesAcronym].append(d)
                        else:
                            series[seriesAcronym] = [d]
            aggLod = []
            for series, eventRecords in series.items():
                # set operation
                ordinals: List[int] = [int(r.get("ordinal"))
                                       for r in eventRecords
                                       if r.get("ordinal")
                                       and ((isinstance(r.get("ordinal"), str) and r.get("ordinal").isnumeric()) or isinstance(r.get("ordinal"), int))]
                if len(ordinals) == 0:
                    continue
                minOrd = min(ordinals)
                maxOrd = max(ordinals)
                numberOfDistinctOrds = len(set(ordinals))
                # count set content
                res = {
                    "series": series,
                    "minOrdinal": minOrd,
                    "maxOrdinal": maxOrd,
                    "avgOrdinal": mean(ordinals),
                    "span": maxOrd-minOrd,
                    "available": numberOfDistinctOrds,
                    "completeness": numberOfDistinctOrds / maxOrd if maxOrd>1 else 1.0
                }
                aggLod.append(res)
            figure=Figure(dataSource.title,caption=f"event series completeness of {dataSource.name}",figLabel=f"esca-{dataSource.name}",sqlQuery=None,fileNames=[histOutputFileName])
            self.figureList.add(figure)
              
            values = [round(record["completeness"], 2) for record in aggLod if isinstance(record["completeness"], float)]
            values.sort()
            threshold =values[len(values)//2]
            h = Histogramm(x=values)
            hps = PlotSettings(outputFile=f"{self.histroot}/{histOutputFileName}", callback=histogrammSettings)
            h.show(xLabel='completeness',
                   yLabel='distribution',
                   title=f'{figure.title}',
                   alpha=self.alpha,
                   density=True,
                   ps=hps,
                   bins=10,
                   vlineAt=threshold)
        
            print(dataSource, len(values), "→", len(values) // 2)
        self.figureList.printAllMarkups()

CEUR-WS

event series completeness of ceurws

Crossref

event series completeness of crossref

dblp

event series completeness of dblp

GND

event series completeness of gnd

OpenResearch

event series completeness of or

OpenResearch

event series completeness of orbackup

OpenResearch

event series completeness of orclone

OpenResearch

event series completeness of orclonebackup

TIBKAT

event series completeness of tibkat

WikiCFP

event series completeness of wikicfp

Wikidata

event series completeness of wikidata