Path: blob/devel/ElmerGUI/PythonQt/src/PythonQtImporter.cpp
3206 views
/*1*2* Copyright (C) 2006 MeVis Research GmbH All Rights Reserved.3*4* This library is free software; you can redistribute it and/or5* modify it under the terms of the GNU Lesser General Public6* License as published by the Free Software Foundation; either7* version 2.1 of the License, or (at your option) any later version.8*9* This library is distributed in the hope that it will be useful,10* but WITHOUT ANY WARRANTY; without even the implied warranty of11* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU12* Lesser General Public License for more details.13*14* Further, this software is distributed without any warranty that it is15* free of the rightful claim of any third person regarding infringement16* or the like. Any license provided herein, whether implied or17* otherwise, applies only to this software file. Patent licenses, if18* any, provided herein do not apply to combinations of this program with19* other software, or any other product whatsoever.20*21* You should have received a copy of the GNU Lesser General Public22* License along with this library; if not, write to the Free Software23* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA24*25* Contact information: MeVis Research GmbH, Universitaetsallee 29,26* 28359 Bremen, Germany or:27*28* http://www.mevis.de29*30*/3132//----------------------------------------------------------------------------------33/*!34// \file PythonQtImporter.h35// \author Florian Link36// \author Last changed by $Author: florian $37// \date 2006-0538*/39// This module was inspired by the zipimport.c module of the original40// Python distribution. Most of the functions are identical or slightly41// modified to do all the loading of Python files via an external file interface.42// In contrast to zipimport.c, this module also writes *.pyc files43// automatically if it has write access/is not inside of a zip file.44//----------------------------------------------------------------------------------4546#include "PythonQtImporter.h"47#include "PythonQtImportFileInterface.h"48#include "PythonQt.h"49#include <QFile>50#include <QFileInfo>5152#define IS_SOURCE 0x053#define IS_BYTECODE 0x154#define IS_PACKAGE 0x25556struct st_mlab_searchorder {57char suffix[14];58int type;59};6061/* mlab_searchorder defines how we search for a module in the Zip62archive: we first search for a package __init__, then for63non-package .pyc, .pyo and .py entries. The .pyc and .pyo entries64are swapped by initmlabimport() if we run in optimized mode. Also,65'/' is replaced by SEP there. */66struct st_mlab_searchorder mlab_searchorder[] = {67{"/__init__.pyc", IS_PACKAGE | IS_BYTECODE},68{"/__init__.pyo", IS_PACKAGE | IS_BYTECODE},69{"/__init__.py", IS_PACKAGE | IS_SOURCE},70{".pyc", IS_BYTECODE},71{".pyo", IS_BYTECODE},72{".py", IS_SOURCE},73{"", 0}74};7576extern PyTypeObject PythonQtImporter_Type;77PyObject *PythonQtImportError;7879QString PythonQtImport::getSubName(const QString& str)80{81int idx = str.lastIndexOf('.');82if (idx!=-1) {83return str.mid(idx+1);84} else {85return str;86}87}8889PythonQtImport::module_info PythonQtImport::getModuleInfo(PythonQtImporter* self, const QString& fullname)90{91QString subname;92struct st_mlab_searchorder *zso;9394subname = getSubName(fullname);95QString path = *self->_path + "/" + subname;9697QString test;98for (zso = mlab_searchorder; *zso->suffix; zso++) {99test = path + zso->suffix;100if (PythonQt::importInterface()->exists(test)) {101if (zso->type & IS_PACKAGE)102return MI_PACKAGE;103else104return MI_MODULE;105}106}107return MI_NOT_FOUND;108}109110111/* PythonQtImporter.__init__112Just store the path argument113*/114int PythonQtImporter_init(PythonQtImporter *self, PyObject *args, PyObject *kwds)115{116self->_path = NULL;117118const char* path;119if (!PyArg_ParseTuple(args, "s",120&path))121return -1;122123if (PythonQt::importInterface()->exists(path)) {124//qDebug("path %s", path);125QString p(path);126const QStringList& ignorePaths = PythonQt::self()->getImporterIgnorePaths();127foreach(QString a, ignorePaths) {128if (a==p) {129PyErr_SetString(PythonQtImportError,130"path ignored");131return -1;132}133}134135self->_path = new QString(p);136137//mlabDebugConst("MLABPython", "PythonQtImporter init: " << *self->_path);138139return 0;140} else {141PyErr_SetString(PythonQtImportError,142"path does not exist error");143return -1;144}145}146147void148PythonQtImporter_dealloc(PythonQtImporter *self)149{150// free the stored path151if (self->_path) delete self->_path;152// free ourself153self->ob_type->tp_free((PyObject *)self);154}155156157/* Check whether we can satisfy the import of the module named by158'fullname'. Return self if we can, None if we can't. */159PyObject *160PythonQtImporter_find_module(PyObject *obj, PyObject *args)161{162PythonQtImporter *self = (PythonQtImporter *)obj;163PyObject *path = NULL;164char *fullname;165166if (!PyArg_ParseTuple(args, "s|O:PythonQtImporter.find_module",167&fullname, &path))168return NULL;169170// mlabDebugConst("MLABPython", "FindModule " << fullname << " in " << *self->_path);171172PythonQtImport::module_info info = PythonQtImport::getModuleInfo(self, fullname);173if (info == PythonQtImport::MI_MODULE || info == PythonQtImport::MI_PACKAGE) {174Py_INCREF(self);175return (PyObject *)self;176} else {177Py_INCREF(Py_None);178return Py_None;179}180}181182/* Load and return the module named by 'fullname'. */183PyObject *184PythonQtImporter_load_module(PyObject *obj, PyObject *args)185{186PythonQtImporter *self = (PythonQtImporter *)obj;187PyObject *code, *mod, *dict;188char *fullname;189QString modpath;190int ispackage;191192if (!PyArg_ParseTuple(args, "s:PythonQtImporter.load_module",193&fullname))194return NULL;195196code = PythonQtImport::getModuleCode(self, fullname, &ispackage, modpath);197if (code == NULL)198return NULL;199200mod = PyImport_AddModule(fullname);201if (mod == NULL) {202Py_DECREF(code);203return NULL;204}205dict = PyModule_GetDict(mod);206207if (PyDict_SetItemString(dict, "__loader__", (PyObject *)self) != 0)208goto error;209210if (ispackage) {211PyObject *pkgpath, *fullpath;212QString subname = PythonQtImport::getSubName(fullname);213int err;214215fullpath = PyString_FromFormat("%s%c%s",216self->_path->toLatin1().constData(),217SEP,218subname.toLatin1().constData());219if (fullpath == NULL)220goto error;221222pkgpath = Py_BuildValue("[O]", fullpath);223Py_DECREF(fullpath);224if (pkgpath == NULL)225goto error;226err = PyDict_SetItemString(dict, "__path__", pkgpath);227Py_DECREF(pkgpath);228if (err != 0)229goto error;230}231mod = PyImport_ExecCodeModuleEx(fullname, code, (char*)modpath.toLatin1().data());232Py_DECREF(code);233if (Py_VerboseFlag)234PySys_WriteStderr("import %s # loaded from %s\n",235fullname, (char*)modpath.toLatin1().data());236return mod;237error:238Py_DECREF(code);239Py_DECREF(mod);240return NULL;241}242243244PyObject *245PythonQtImporter_get_data(PyObject *obj, PyObject *args)246{247// EXTRA, NOT YET IMPLEMENTED248return NULL;249}250251PyObject *252PythonQtImporter_get_code(PyObject *obj, PyObject *args)253{254PythonQtImporter *self = (PythonQtImporter *)obj;255char *fullname;256257if (!PyArg_ParseTuple(args, "s:PythonQtImporter.get_code", &fullname))258return NULL;259260QString notused;261return PythonQtImport::getModuleCode(self, fullname, NULL, notused);262}263264PyObject *265PythonQtImporter_get_source(PyObject *obj, PyObject *args)266{267// EXTRA, NOT YET IMPLEMENTED268/*269PythonQtImporter *self = (PythonQtImporter *)obj;270PyObject *toc_entry;271char *fullname, *subname, path[MAXPATHLEN+1];272int len;273enum module_info mi;274275if (!PyArg_ParseTuple(args, "s:PythonQtImporter.get_source", &fullname))276return NULL;277278mi = get_module_info(self, fullname);279if (mi == MI_ERROR)280return NULL;281if (mi == MI_NOT_FOUND) {282PyErr_Format(PythonQtImportError, "can't find module '%.200s'",283fullname);284return NULL;285}286subname = get_subname(fullname);287288len = make_filename(PyString_AsString(self->prefix), subname, path);289if (len < 0)290return NULL;291292if (mi == MI_PACKAGE) {293path[len] = SEP;294strcpy(path + len + 1, "__init__.py");295}296else297strcpy(path + len, ".py");298299toc_entry = PyDict_GetItemString(self->files, path);300if (toc_entry != NULL)301return get_data(PyString_AsString(self->archive), toc_entry);302303Py_INCREF(Py_None);304return Py_None;305*/306return NULL;307}308309PyDoc_STRVAR(doc_find_module,310"find_module(fullname, path=None) -> self or None.\n\311\n\312Search for a module specified by 'fullname'. 'fullname' must be the\n\313fully qualified (dotted) module name. It returns the PythonQtImporter\n\314instance itself if the module was found, or None if it wasn't.\n\315The optional 'path' argument is ignored -- it's there for compatibility\n\316with the importer protocol.");317318PyDoc_STRVAR(doc_load_module,319"load_module(fullname) -> module.\n\320\n\321Load the module specified by 'fullname'. 'fullname' must be the\n\322fully qualified (dotted) module name. It returns the imported\n\323module, or raises PythonQtImportError if it wasn't found.");324325PyDoc_STRVAR(doc_get_data,326"get_data(pathname) -> string with file data.\n\327\n\328Return the data associated with 'pathname'. Raise IOError if\n\329the file wasn't found.");330331PyDoc_STRVAR(doc_get_code,332"get_code(fullname) -> code object.\n\333\n\334Return the code object for the specified module. Raise PythonQtImportError\n\335is the module couldn't be found.");336337PyDoc_STRVAR(doc_get_source,338"get_source(fullname) -> source string.\n\339\n\340Return the source code for the specified module. Raise PythonQtImportError\n\341is the module couldn't be found, return None if the archive does\n\342contain the module, but has no source for it.");343344PyMethodDef PythonQtImporter_methods[] = {345{"find_module", PythonQtImporter_find_module, METH_VARARGS,346doc_find_module},347{"load_module", PythonQtImporter_load_module, METH_VARARGS,348doc_load_module},349{"get_data", PythonQtImporter_get_data, METH_VARARGS,350doc_get_data},351{"get_code", PythonQtImporter_get_code, METH_VARARGS,352doc_get_code},353{"get_source", PythonQtImporter_get_source, METH_VARARGS,354doc_get_source},355{NULL, NULL} /* sentinel */356};357358359PyDoc_STRVAR(PythonQtImporter_doc,360"PythonQtImporter(path) -> PythonQtImporter object\n\361\n\362Create a new PythonQtImporter instance. 'path' must be a valid path on disk/or inside of a zip file known to MeVisLab\n\363. Every path is accepted.");364365#define DEFERRED_ADDRESS(ADDR) 0366367PyTypeObject PythonQtImporter_Type = {368PyObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type))3690,370"PythonQtImport.PythonQtImporter",371sizeof(PythonQtImporter),3720, /* tp_itemsize */373(destructor)PythonQtImporter_dealloc, /* tp_dealloc */3740, /* tp_print */3750, /* tp_getattr */3760, /* tp_setattr */3770, /* tp_compare */3780, /* tp_repr */3790, /* tp_as_number */3800, /* tp_as_sequence */3810, /* tp_as_mapping */3820, /* tp_hash */3830, /* tp_call */3840, /* tp_str */385PyObject_GenericGetAttr, /* tp_getattro */3860, /* tp_setattro */3870, /* tp_as_buffer */388Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE , /* tp_flags */389PythonQtImporter_doc, /* tp_doc */3900, /* tp_traverse */3910, /* tp_clear */3920, /* tp_richcompare */3930, /* tp_weaklistoffset */3940, /* tp_iter */3950, /* tp_iternext */396PythonQtImporter_methods, /* tp_methods */3970, /* tp_members */3980, /* tp_getset */3990, /* tp_base */4000, /* tp_dict */4010, /* tp_descr_get */4020, /* tp_descr_set */4030, /* tp_dictoffset */404(initproc)PythonQtImporter_init, /* tp_init */405PyType_GenericAlloc, /* tp_alloc */406PyType_GenericNew, /* tp_new */407PyObject_Del, /* tp_free */408};409410411/* Given a buffer, return the long that is represented by the first4124 bytes, encoded as little endian. This partially reimplements413marshal.c:r_long() */414long415PythonQtImport::getLong(unsigned char *buf)416{417long x;418x = buf[0];419x |= (long)buf[1] << 8;420x |= (long)buf[2] << 16;421x |= (long)buf[3] << 24;422#if SIZEOF_LONG > 4423/* Sign extension for 64-bit machines */424x |= -(x & 0x80000000L);425#endif426return x;427}428429FILE *430open_exclusive(const QString& filename)431{432#if defined(O_EXCL)&&defined(O_CREAT)&&defined(O_WRONLY)&&defined(O_TRUNC)433/* Use O_EXCL to avoid a race condition when another process tries to434write the same file. When that happens, our open() call fails,435which is just fine (since it's only a cache).436XXX If the file exists and is writable but the directory is not437writable, the file will never be written. Oh well.438*/439QFile::remove(filename);440441int fd;442int flags = O_EXCL|O_CREAT|O_WRONLY|O_TRUNC;443#ifdef O_BINARY444flags |= O_BINARY; /* necessary for Windows */445#endif446#ifdef WIN32447fd = _wopen(filename.ucs2(), flags, 0666);448#else449fd = open(filename.local8Bit(), flags, 0666);450#endif451if (fd < 0)452return NULL;453return fdopen(fd, "wb");454#else455/* Best we can do -- on Windows this can't happen anyway */456return fopen(filename.toLocal8Bit().constData(), "wb");457#endif458}459460461void PythonQtImport::writeCompiledModule(PyCodeObject *co, const QString& filename, long mtime)462{463FILE *fp;464465fp = open_exclusive(filename);466if (fp == NULL) {467if (Py_VerboseFlag)468PySys_WriteStderr(469"# can't create %s\n", filename.toLatin1().constData());470return;471}472#if PY_VERSION_HEX < 0x02040000473PyMarshal_WriteLongToFile(PyImport_GetMagicNumber(), fp);474#else475PyMarshal_WriteLongToFile(PyImport_GetMagicNumber(), fp, Py_MARSHAL_VERSION);476#endif477/* First write a 0 for mtime */478#if PY_VERSION_HEX < 0x02040000479PyMarshal_WriteLongToFile(0L, fp);480#else481PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);482#endif483#if PY_VERSION_HEX < 0x02040000484PyMarshal_WriteObjectToFile((PyObject *)co, fp);485#else486PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION);487#endif488if (ferror(fp)) {489if (Py_VerboseFlag)490PySys_WriteStderr("# can't write %s\n", filename.toLatin1().constData());491/* Don't keep partial file */492fclose(fp);493QFile::remove(filename);494return;495}496/* Now write the true mtime */497fseek(fp, 4L, 0);498#if PY_VERSION_HEX < 0x02040000499PyMarshal_WriteLongToFile(mtime, fp);500#else501PyMarshal_WriteLongToFile(mtime, fp, Py_MARSHAL_VERSION);502#endif503fflush(fp);504fclose(fp);505if (Py_VerboseFlag)506PySys_WriteStderr("# wrote %s\n", filename.toLatin1().constData());507//#ifdef macintosh508// PyMac_setfiletype(cpathname, 'Pyth', 'PYC ');509//#endif510}511512/* Given the contents of a .py[co] file in a buffer, unmarshal the data513and return the code object. Return None if it the magic word doesn't514match (we do this instead of raising an exception as we fall back515to .py if available and we don't want to mask other errors).516Returns a new reference. */517PyObject *518PythonQtImport::unmarshalCode(const QString& path, const QByteArray& data, time_t mtime)519{520PyObject *code;521// ugly cast, but Python API is not const safe522char *buf = (char*) data.constData();523int size = data.size();524525if (size <= 9) {526PySys_WriteStderr("# %s has bad pyc data\n",527path.toLatin1().constData());528Py_INCREF(Py_None);529return Py_None;530}531532if (getLong((unsigned char *)buf) != PyImport_GetMagicNumber()) {533if (Py_VerboseFlag)534PySys_WriteStderr("# %s has bad magic\n",535path.toLatin1().constData());536Py_INCREF(Py_None);537return Py_None;538}539540if (mtime != 0 && !(getLong((unsigned char *)buf + 4) == mtime)) {541if (Py_VerboseFlag)542PySys_WriteStderr("# %s has bad mtime\n",543path.toLatin1().constData());544Py_INCREF(Py_None);545return Py_None;546}547548code = PyMarshal_ReadObjectFromString(buf + 8, size - 8);549if (code == NULL)550return NULL;551if (!PyCode_Check(code)) {552Py_DECREF(code);553PyErr_Format(PyExc_TypeError,554"compiled module %.200s is not a code object",555path.toLatin1().constData());556return NULL;557}558return code;559}560561562/* Given a string buffer containing Python source code, compile it563return and return a code object as a new reference. */564PyObject *565PythonQtImport::compileSource(const QString& path, const QByteArray& data)566{567PyObject *code;568QByteArray data1 = data;569// in qt4, data is null terminated570// data1.resize(data.size()+1);571// data1.data()[data.size()-1] = 0;572code = Py_CompileString(data.data(), path.toLatin1().constData(),573Py_file_input);574return code;575}576577578/* Return the code object for the module named by 'fullname' from the579Zip archive as a new reference. */580PyObject *581PythonQtImport::getCodeFromData(const QString& path, int isbytecode,int ispackage, time_t mtime)582{583bool hasImporter = PythonQt::importInterface()!=NULL;584585PyObject *code;586587QByteArray qdata;588if (!hasImporter) {589QFile file(path);590QIODevice::OpenMode flags = QIODevice::ReadOnly;591if (!isbytecode) {592flags |= QIODevice::Text;593}594if (!file.open(flags)) {595return NULL;596}597qdata = file.readAll();598} else {599if (!isbytecode) {600// mlabDebugConst("MLABPython", "reading source " << path);601bool ok;602qdata = PythonQt::importInterface()->readSourceFile(path, ok);603if (!ok) {604// mlabErrorConst("PythonQtImporter","File could not be verified" << path);605return NULL;606}607if (qdata == " ") {608qdata.clear();609}610} else {611qdata = PythonQt::importInterface()->readFileAsBytes(path);612}613}614615if (isbytecode) {616// mlabDebugConst("MLABPython", "reading bytecode " << path);617code = unmarshalCode(path, qdata, mtime);618}619else {620// mlabDebugConst("MLABPython", "compiling source " << path);621code = compileSource(path, qdata);622// save a pyc file if possible623QDateTime time;624time = hasImporter?PythonQt::importInterface()->lastModifiedDate(path):QFileInfo(path).lastModified();625writeCompiledModule((PyCodeObject*)code, path+"c", time.toTime_t());626}627return code;628}629630time_t631PythonQtImport::getMTimeOfSource(const QString& path)632{633time_t mtime = 0;634QString path2 = path;635path2.truncate(path.length()-1);636if (PythonQt::importInterface()->exists(path2)) {637mtime = PythonQt::importInterface()->lastModifiedDate(path2).toTime_t();638}639return mtime;640}641642/* Get the code object associated with the module specified by643'fullname'. */644PyObject *645PythonQtImport::getModuleCode(PythonQtImporter *self, char *fullname,646int *p_ispackage, QString& modpath)647{648QString subname;649struct st_mlab_searchorder *zso;650651subname = getSubName(fullname);652QString path = *self->_path + "/" + subname;653654QString test;655for (zso = mlab_searchorder; *zso->suffix; zso++) {656PyObject *code = NULL;657test = path + zso->suffix;658659if (Py_VerboseFlag > 1)660PySys_WriteStderr("# trying %s\n",661test.toLatin1().constData());662if (PythonQt::importInterface()->exists(test)) {663time_t mtime = 0;664int ispackage = zso->type & IS_PACKAGE;665int isbytecode = zso->type & IS_BYTECODE;666667if (isbytecode)668mtime = getMTimeOfSource(test);669if (p_ispackage != NULL)670*p_ispackage = ispackage;671code = getCodeFromData(test, isbytecode, ispackage, mtime);672if (code == Py_None) {673Py_DECREF(code);674continue;675}676if (code != NULL)677modpath = test;678return code;679}680}681PyErr_Format(PythonQtImportError, "can't find module '%.200s'", fullname);682683return NULL;684}685686QString PythonQtImport::replaceExtension(const QString& str, const QString& ext)687{688QString r;689int i = str.lastIndexOf('.');690if (i!=-1) {691r = str.mid(0,i) + "." + ext;692} else {693r = str + "." + ext;694}695return r;696}697698PyObject* PythonQtImport::getCodeFromPyc(const QString& file)699{700bool hasImporter = PythonQt::importInterface()!=NULL;701702PyObject* code;703const static QString pycStr("pyc");704QString pyc = replaceExtension(file, pycStr);705if ((hasImporter && PythonQt::importInterface()->exists(pyc)) ||706(!hasImporter && QFile::exists(pyc))) {707time_t mtime = 0;708mtime = getMTimeOfSource(pyc);709code = getCodeFromData(pyc, true, false, mtime);710if (code != Py_None && code != NULL) {711return code;712}713if (code) {714Py_DECREF(code);715}716}717code = getCodeFromData(file,false,false,0);718return code;719}720721/* Module init */722723PyDoc_STRVAR(mlabimport_doc,724"Imports python files into MeVisLab, completely replaces internal python import");725726void PythonQtImport::init()727{728PyObject *mod;729730if (PyType_Ready(&PythonQtImporter_Type) < 0)731return;732733/* Correct directory separator */734mlab_searchorder[0].suffix[0] = SEP;735mlab_searchorder[1].suffix[0] = SEP;736mlab_searchorder[2].suffix[0] = SEP;737if (Py_OptimizeFlag) {738/* Reverse *.pyc and *.pyo */739struct st_mlab_searchorder tmp;740tmp = mlab_searchorder[0];741mlab_searchorder[0] = mlab_searchorder[1];742mlab_searchorder[1] = tmp;743tmp = mlab_searchorder[3];744mlab_searchorder[3] = mlab_searchorder[4];745mlab_searchorder[4] = tmp;746}747748mod = Py_InitModule4("PythonQtImport", NULL, mlabimport_doc,749NULL, PYTHON_API_VERSION);750751PythonQtImportError = PyErr_NewException("PythonQtImport.PythonQtImportError",752PyExc_ImportError, NULL);753if (PythonQtImportError == NULL)754return;755756Py_INCREF(PythonQtImportError);757if (PyModule_AddObject(mod, "PythonQtImportError",758PythonQtImportError) < 0)759return;760761Py_INCREF(&PythonQtImporter_Type);762if (PyModule_AddObject(mod, "PythonQtImporter",763(PyObject *)&PythonQtImporter_Type) < 0)764return;765766// set our importer into the path_hooks to handle all path on sys.path767PyObject* classobj = PyDict_GetItemString(PyModule_GetDict(mod), "PythonQtImporter");768PyObject* path_hooks = PySys_GetObject("path_hooks");769PyList_Append(path_hooks, classobj);770}771772773