משתמש:ערן/HarvestLabel.js

/*
Harvest labels
=====================
This is EXPERIMENTAL script for quickly adding labels to wikidata. 
The script queries wikidata for missing labels, and uses infoboxes to suggest labels, and allow users to easily add labels.

Author: [[:he:User:ערן]]
*/


if (mw.config.get('wgNamespaceNumber') === 10) mw.loader.using( [ 'oojs-ui-windows', 'jquery.wikibase.linkitem', 'mw.config.values.wbRepo' ], function(){
var INFOBOX_TO_ENTITIES_MAP_PAGE = 'ויקיפדיה:ויקינתונים/תבניות מידע';
mw.messages.set({
	'qlabel-adder-title' : 'הוספת תוויות',
	'qlabel-save-continue' : 'שמירה והמשך',
	'qlabel-skip' : 'דילוג',
	'qlabel-skip-page' : 'דילוג דף',
	'qlbael-sidelink': 'הוספת תוויות',
	'qlabel-person-sidelink': 'הוספת תווית לאישים',
	'qlabel-prize-sidelink': 'הוספת תווית לפרסים',
	'qlabel-save-failed': 'השמירה נכשלה ;( $1', 
	'qlabel-save-success': 'השמירה הצליחה',
	'qlabel-no-results': 'לא נמצאו דפים מתאימים',
	'qlabel-no-parameter-results': 'לא קיים מיפוי לפרמטר בשם $1',
	'qlabel-no-templatedata': 'לא קיימת מפת wikidata בתבנית',
	'qlabel-request-paramname': 'שם הפרמטר בתבנית',
	'qlabel-locallabel-placeholder': 'תווית מתורגמת',
	'qlabel-fields-title': 'הוספת תוויות',
	'qlabel-english-label': 'תווית באנגלית',
	'qlabel-local-label': 'תווית בעברית',
	'qlabel-local-source': 'מבוסס על',
	'qlabel-infobox-missing-mapping': 'חסר מיפוי של התבנית ליישויות בוויקינתונים. נא להוסיף ב$1'
});

function HarvestLabelDialog( config ) {
	var repoConfig = mw.config.get( 'wbRepo' );
	this.extractorRgx = null;
	this.currentEntity = null;
	this.currentSource = null;
	this.translatedEntities = {};
	this.ignoreSources = {};
	this.wikidataApi = new wikibase.api.RepoApi( wikibase.api.getLocationAgnosticMwApi(repoConfig.url + repoConfig.scriptPath + '/api.php') );
	HarvestLabelDialog.super.call( this, config );
}
OO.inheritClass( HarvestLabelDialog, OO.ui.ProcessDialog ); 

// Specify a name for .addWindows()
HarvestLabelDialog.static.name = 'HarvestLabelDialog';
// Specify a title statically (or, alternatively, with data passed to the opening() method). 
HarvestLabelDialog.static.title = mw.msg('qlabel-adder-title');

HarvestLabelDialog.static.actions = [
  { action: 'saveContinue', label: mw.msg('qlabel-save-continue'), flags: [ 'other', 'constructive' ] },
  { action: 'skipOne', label: mw.msg('qlabel-skip'), flags: [ 'other', 'progressive' ] },
  { action: 'skipPage', label: mw.msg('qlabel-skip-page'), flags: [ 'other', 'progressive' ] },
  { label: 'Cancel', flags: 'safe' }
];


HarvestLabelDialog.prototype.initialize = function () {
	HarvestLabelDialog.super.prototype.initialize.call( this );
	this.content = new OO.ui.PanelLayout( { padded: true, expanded: true } );
  
	this.sourceLabel = new OO.ui.LabelWidget( { 
		label: ''
	} );

	this.localLabel = new OO.ui.TextInputWidget( { 
		placeholder: mw.msg('qlabel-locallabel-placeholder')
	} );

	this.localSource = new OO.ui.LabelWidget( {
	  label: ''
	} );
	this.fieldset = new OO.ui.FieldsetLayout( { 
		label: mw.msg('qlabel-fields-title'),
		classes: ["container"]
	} );
	this.fieldset.addItems( [ 
		new OO.ui.FieldLayout( this.sourceLabel, {
			label: mw.msg('qlabel-english-label')
		} ),
		new OO.ui.FieldLayout( this.localLabel, { 
			label: mw.msg('qlabel-local-label')
		} ),
		new OO.ui.FieldLayout( this.localSource, { 
			label: mw.msg('qlabel-local-source')
		} )
	] );

	this.content.$element.append( this.fieldset.$element );
	this.$body.append( this.content.$element );
};
  
HarvestLabelDialog.prototype.getBodyHeight = function () {
  return 400;
};

HarvestLabelDialog.prototype.getSetupProcess = function ( data ) {
	return HarvestLabelDialog.super.prototype.getSetupProcess.call( this, data )
		.next( function () {
			this.dataSource = data;
			this.nextLabel(false);
		}, this );
};

HarvestLabelDialog.prototype.getActionProcess = function ( action ) {
  var dialog = this;
  switch ( action ) {
	case 'skipOne':
		return new OO.ui.Process( function () {
		  dialog.nextLabel(false);
		}, this );
	case 'skipPage':
		return new OO.ui.Process( function () {
		  dialog.nextLabel(true);
		}, this );
	case 'saveContinue':
		return new OO.ui.Process( function () {
		  dialog.saveLabel();
		}, this );
  }

  // Fallback to parent handler.
  return HarvestLabelDialog.super.prototype.getActionProcess.call( this, action );
};

HarvestLabelDialog.prototype.saveLabel = function ( ) {
	if (this.currentEntity)
	{
		this.translatedEntities[this.currentEntity] = 1; //mark as used
		var saveResponse = this.wikidataApi.setLabel(this.currentEntity, 0, this.localLabel.getValue(), mw.config.get('wgContentLanguage'));
		saveResponse.done(function(d) {
			if (d && d.success)	mw.notify(mw.msg('qlabel-save-success'));
			else if (d && d.error) {
				mw.notify(mw.msg('qlabel-save-failed: $1', d.error.info));
			}
		});
	}
	this.translatedEntities[this.currentEntity] = 1;
	this.nextLabel(false);
}

HarvestLabelDialog.prototype.nextLabel = function ( removePage ) {	
	var self = this;
	if ( removePage ) this.ignoreSources[this.currentSource] = true; // remove the current page as valid source
	this.dataSource.getNext().done(function(d){
		self.currentEntity = d.entity;
		self.currentSource = d.localSourcePlain;
		if (self.translatedEntities[self.currentEntity] || self.ignoreSources[d.localSourcePlain]) {
			self.nextLabel(false);
			return;
		}
		
		self.localLabel.setValue(d.localLabel);
		self.localSource.setLabel(d.localSource);
		self.sourceLabel.setLabel(d.sourceLabel);		
	}).fail(function(){
		self.close();
	});
}

function createHarvestLabelDialog(d) {
	// Make the window.
	var qLabelAdder = new HarvestLabelDialog( {
	  size: 'medium'
	} );

	// Create and append a window manager, which will open and close the window. 
	var windowManager = new OO.ui.WindowManager();
	$( 'body' ).append( windowManager.$element );

	// Add the window to the window manager using the addWindows() method.
	windowManager.addWindows( [ qLabelAdder ] );

	// Open the window!
	windowManager.openWindow( qLabelAdder, d );
}

function runSparql(query) {
	return $.getJSON('https://query.wikidata.org/sparql?format=json&query=' + query);
}

function queryMissingLabels(entityProp, entityVal, claim)
{
	var query = "SELECT DISTINCT ?item ?sitelink ?linkedprop ?enlabel WHERE { \
 ?item wdt:" + entityProp.split('/').join('/wdt:') + " wd:"+entityVal+" .\
 ?item p:"+claim+" ?statement .\
 ?statement ps:"+claim+" ?linkedprop .\
 ?sitelink schema:about ?item .\
 ?sitelink schema:inLanguage '"+mw.config.get('wgContentLanguage')+"' \
 MINUS {?linkedprop rdfs:label ?locallabel filter(lang(?locallabel) = '"+mw.config.get('wgContentLanguage')+"')}\
 OPTIONAL {?linkedprop rdfs:label ?enlabel filter(lang(?enlabel) = 'en')}\
 SERVICE wikibase:label { bd:serviceParam wikibase:language '"+mw.config.get('wgContentLanguage')+"'	}  \
}";
	return runSparql(query);
}




function createPersonLabelSource() {
	var Qlang = 'Q1860', lang = 'en'; //en
	
	var query='SELECT ?item ?itemLang ?name WHERE {\
			  ?item wdt:P31 wd:Q5.\
			  ?item wdt:P735 ?first.\
			  ?item wdt:P734 ?last.\
			  ?item wdt:P27 ?country.\
			  ?country wdt:P37 wd:'+Qlang+'.  \
			  ?item rdfs:label ?itemLang filter (lang(?itemLang) = "'+lang+'")\
			  optional {?item rdfs:label ?itemLabelHE filter (lang(?itemLabelHE) = "he")}.\
			  FILTER (!BOUND(?itemLabelHE)).\
			  ?first rdfs:label ?firstLabelHE filter (lang(?firstLabelHE) = "he").\
			  ?last rdfs:label ?lastLabelHE filter (lang(?lastLabelHE) = "he").\
			  BIND(CONCAT(?firstLabelHE, " ", ?lastLabelHE) AS ?name)\
			  } LIMIT 100';

	var dfd = new jQuery.Deferred();
	runSparql(query).done(function(d) {
		dfd.resolve(new PersonLabelSource(d));
	});
	return dfd;
}

function PersonLabelSource(data) {
	this.data = data.results.bindings;
	this.index=-1;
}


PersonLabelSource.prototype.getNext = function () {
	var dfd = new $.Deferred();
	this.index++;
	if(this.index >= this.data.length)
	{
		dfd.reject();
		return dfd;
	}
	
	dfd.resolve({
		entity: /Q[0-9]+/.exec(this.data[this.index].item.value)[0],
		localLabel: this.data[this.index].name.value,
		localSource: $('<a href="'+this.data[this.index].item.value+'" target="_blank">'+this.data[this.index].itemLang.value+'</a>' ),
		localSourcePlain: this.data[this.index].itemLang.value,  
		sourceLabel: $('<a href="'+this.data[this.index].item.value+'" target="_blank">'+this.data[this.index].itemLang.value+'</a>' )
	});
	return dfd;
}

function createPrizeLabelSource() {
	var query=`SELECT ?item ?itemLabel ?nameBy ?nameByLabel WHERE 
		{
		  ?item wdt:P31/wdt:P279* wd:Q618779.
		  ?item wdt:P138 ?nameBy.
		  FILTER( EXISTS {
		   ?nameBy rdfs:label ?lang_label_nameBy.
		   FILTER(LANG(?lang_label_nameBy) = "he")
		 }) .
		  FILTER(NOT EXISTS {
		   ?item rdfs:label ?lang_label.
		   FILTER(LANG(?lang_label) = "he")
		 })
		  SERVICE wikibase:label { bd:serviceParam wikibase:language "he,en,fr,de". }
		}`;

	var dfd = new jQuery.Deferred();
	runSparql(query).done(function(d) {
		dfd.resolve(new PrizeLabelSource(d));
	});
	return dfd;
}

function PrizeLabelSource(data) {
	this.data = data.results.bindings;
	this.index=-1;
}


PrizeLabelSource.prototype.getNext = function () {
	var dfd = new $.Deferred();
	this.index++;
	if(this.index >= this.data.length)
	{
		dfd.reject();
		return dfd;
	}
	
	dfd.resolve({
		entity: /Q[0-9]+/.exec(this.data[this.index].item.value)[0],
		localLabel: this.data[this.index].nameByLabel.value,
		localSource: $('<a href="'+this.data[this.index].nameBy.value+'" target="_blank">'+this.data[this.index].nameByLabel.value+'</a>' ),
		localSourcePlain: this.data[this.index].nameByLabel.value,  
		sourceLabel: $('<a href="'+this.data[this.index].item.value+'" target="_blank">'+this.data[this.index].itemLabel.value+'</a>' )
	});
	return dfd;
}

function getTemplateToWikidataMap() {
	var dfd = new jQuery.Deferred();
	new mw.Api().get({ action:'parse', page: INFOBOX_TO_ENTITIES_MAP_PAGE, prop: 'wikitext' }).done(function(d) {
		var text = d.parse.wikitext['*'],
			extractor = /\{\{.+?\|(.+?)\|(.+?)\|(.+?)\}\}/g,
			templateToWikidata = {}, m;

		while(m = extractor.exec(text)){
			templateToWikidata[m[3]] = [m[1], m[2]];
		}
		if (templateToWikidata[mw.config.get('wgPageName')]) {
			dfd.resolve(templateToWikidata[mw.config.get('wgPageName')]);
			return;
		}
		alert(mw.msg('qlabel-infobox-missing-mapping', INFOBOX_TO_ENTITIES_MAP_PAGE));
		dfd.reject();
	});
	return dfd;
}

function findParamProperty(paramName) { 
	var dfd = new jQuery.Deferred(),
	    api = new mw.Api();
	api.get({
		action: 'templatedata',
		titles: mw.config.get('wgPageName'),
		redirects: 1
	}).done(function(data) {
		var templatedata = {};
		for (var pageid in data.pages) {
            templatedata = data.pages[pageid];
        }

	if (!templatedata.maps || ! templatedata.maps.wikidata) {
		alert(mw.msg('qlabel-no-templatedata'));
		dfd.reject();
		return;
	}

	for(var wikidataProp in templatedata.maps.wikidata){
		if (paramName == templatedata.maps.wikidata[wikidataProp]){
			dfd.resolve(wikidataProp);
			return;
		}
	}

	alert(mw.msg('qlabel-no-parameter-results', paramName));
		dfd.reject();
	});
	return dfd;
}

/*
 * Harvest labels from templates
*/
function createHarvestLabelsSource()
{
	var dfd = jQuery.Deferred();
	var templateParam = prompt(mw.msg('qlabel-request-paramname'));
	if (!templateParam) return;
	$.when(findParamProperty(templateParam), getTemplateToWikidataMap()).done(function(wikidataProp, wikidataMap){
		queryMissingLabels(wikidataMap[0], wikidataMap[1], wikidataProp).done(function(d) {
			var localTitles = [],
				titlesToEntities={};
			 $.each(d.results.bindings, function(i, e){
				 var title = decodeURIComponent(e.sitelink.value.split('/wiki/')[1]);
				 title = new mw.Title(title).getNameText(); // replace _ to space similar to Wikipedia API responses
				 localTitles.push(title);
				 var linkedEntity = e.linkedprop.value.split('/entity/')[1];
				 var enLabel = (e.enlabel && e.enlabel && e.enlabel.value)?  e.enlabel.value : linkedEntity;
				 if (titlesToEntities.hasOwnProperty(title)) {
					titlesToEntities[title].push({ 'entity': linkedEntity, 'entityLabel': enLabel });
				 } else {
					 titlesToEntities[title] = [{ 'entity': linkedEntity, 'entityLabel': enLabel }];
				 }
			 });
			 if (localTitles.length == 0) {
				 mw.notify(mw.msg('qlabel-no-results'));
				return;
			 }
			var wikidataLabels = { 
				templateParamName: templateParam,
				localTitles: localTitles,
				titlesToEntities: titlesToEntities 
			};
			
			dfd.resolve(new HarvestLabelsSource(wikidataLabels, wikidataProp));
		});				
	});
	return dfd;
}


// HarvestLabelsSource
function HarvestLabelsSource(data, prop) {
	this.property = prop;
	this.templateParamName = data.templateParamName;
	this.localTitles = data.localTitles;
	this.titlesToEntities = data.titlesToEntities;

	//this.extractorRgx = new RegExp(this.templateParamName + ' *=\s*([^=]+)');
	this.extractorRgx = new RegExp(this.templateParamName + ' *=\s*([^=]+)(?=\\|.+=|\}\})');
	this.currentBatch = 0;
	this.batchSize = 20;
	this.pageI = -1;
	this.pageClaimI = -1;
	this.suggestedTranslateVals = {}
	this.phase = 'label-with-suggestions';
}

HarvestLabelsSource.prototype.suggestLabels = function () {
	var self = this, dfd = new $.Deferred();
	if (this.localTitles.length < this.currentBatch) return dfd.resolve();
	new mw.Api().get({
		action: 'query',
		prop: 'revisions',
		titles: this.localTitles.slice(this.currentBatch, this.currentBatch + this.batchSize).join('|'),
		rvprop: 'content',
		indexpageids: 1
	 }).done(function(d){
		self.currentBatch += self.batchSize;
		for (var i = 0; i < d.query.pageids.length; i++) {
			 var pageid = d.query.pageids[i];
			 if (pageid < 0) continue;
			 var pageText = d.query.pages[pageid].revisions[0]['*'];
			 var templateVal = self.extractorRgx.exec(pageText);
			 if (!templateVal || templateVal[1].trim().length === 0 || templateVal[1].trim() == '-') continue;
			 templateVal = templateVal[1].replace(/'''/g, '').replace(/\n/g, ' ').replace(/\[\[([^|\]]+?)\]\]/g, '$1').replace(/,?{{ש}}/g, ', ').trim();
			 if (/\|.+= *$/.test(templateVal)) continue;
			
			self.suggestedTranslateVals[d.query.pages[pageid].title] = templateVal;
			if (templateVal.split(', ').length > 3) {
				// filter it if possible
				new mw.Api().get({
					action: 'parse',
					title: d.query.pages[pageid].title,
					text: '{{#property:' + self.property + '}}',
					prop: 'text',
					disablelimitreport: 1,
					wrapoutputclass: ''
				}).done(function(dd){
					var suggestedVals = self.suggestedTranslateVals[dd.parse.title];
					$(dd.parse.text['*']).text().trim().split(',').forEach(r=> suggestedVals = suggestedVals.replace(r,''));
					suggestedVals = suggestedVals.replace(/, ?,/g, ',').replace(/[, ]+$/, '');
					if (suggestedVals.length)
						self.suggestedTranslateVals[dd.parse.title] = suggestedVals;
					else
						delete self.suggestedTranslateVals[dd.parse.title];
				});
			}
		}
		dfd.resolve();
	 });
	return dfd;
}

HarvestLabelsSource.prototype.getNext = function () {
	var dfd = new $.Deferred();

	// pre-fetch
	if ( (this.phase == 'label-with-suggestions') && (this.currentBatch < this.pageI+5) && this.currentBatch<this.localTitles.length) {
		var self = this;
		this.suggestLabels().done(function(){
			self.getNext().done(function(d){
				dfd.resolve(d);
			}).fail(function(){
				dfd.reject();
			});
		});

		return dfd;
	}

	if (this.pageI==-1) {
		this.pageI = 0;
		this.pageClaimI = 0;
	}
	else {
		if (this.pageClaimI+1 < this.titlesToEntities[this.localTitles[this.pageI]].length){
			this.pageClaimI++;
		} else {
			this.pageI++;
			this.pageClaimI = 0;
		}
	}

	if (this.pageI === this.localTitles.length)
	{
		if (this.phase == 'label-without-suggestions') {
			dfd.reject(); // no more work
			return dfd;
		} else {
			// move phase
			this.phase = 'label-without-suggestions';
			this.pageI = -1;
			this.pageClaim = 0;
			return this.getNext();
		}
	}

	var entityLabelData = this.titlesToEntities[this.localTitles[this.pageI]][this.pageClaimI];
	var localLabel = '';
	if(this.phase == 'label-with-suggestions') {
		var hasSuggestion = this.suggestedTranslateVals.hasOwnProperty(this.localTitles[this.pageI]);
		if (!hasSuggestion) {
			return this.getNext();
		}
		
		localLabel = this.suggestedTranslateVals[this.localTitles[this.pageI]];
	}

	dfd.resolve({
		entity: entityLabelData.entity,
		localLabel: localLabel,
		localSource: $('<a href="/wiki/'+encodeURI(this.localTitles[this.pageI])+'" target="_blank">'+this.localTitles[this.pageI]+'</a>' ),
		localSourcePlain: this.localTitles[this.pageI],  
		sourceLabel: $('<a href="//www.wikidata.org/wiki/'+entityLabelData.entity+'" target="_blank">'+entityLabelData.entityLabel+'</a>' )
	});
	return dfd;
}

// end of HarvestLabelsSource

function createaddLabelsButton(){
	return $(mw.util.addPortletLink(
					'p-tb',
					'#',
					mw.msg('qlbael-sidelink'),
					't-harvestlabel',
					mw.msg('qlabel-invoke'),
					null,
					'#t-whatlinkshere'
	)).click(function(e){
		createHarvestLabelsSource().done(function( d ) {
			mw.loader.using('oojs-ui-windows', function() { createHarvestLabelDialog(d); } );
		});
		e.preventDefault();
	});
}


function createGeneratorTranslationButton(){
	var translateGenerator =  {
		'prize': createPrizeLabelSource, 
		'person': createPersonLabelSource
	}
	for (var obj in translateGenerator){
		$(mw.util.addPortletLink(
				'p-tb',
				'#',
				mw.msg(`qlabel-${obj}-sidelink`),
				`t-add${obj}label`,
				mw.msg(`qlabel-${obj}-sidelink`),
				null,
				'#t-whatlinkshere'
		)).click(function(e){
			$(this).data('generator').call(null).done(function( d ) {
				mw.loader.using('oojs-ui-windows', function() { createHarvestLabelDialog(d); } );
			});
			e.preventDefault();
		}).data('generator', translateGenerator[obj]);
	}

}

createaddLabelsButton();
createGeneratorTranslationButton();

});