Kettle插件机制

一、插件加载过程分析

Spoon.java中main函数

public static void main( String[] a ) throws KettleException 
{
 Future<KettleException> pluginRegistryFuture = executor.submit( new Callable<KettleException>() {
      @Override
      public KettleException call() throws Exception {
        registerUIPluginObjectTypes();
        KettleClientEnvironment.getInstance().setClient( KettleClientEnvironment.ClientType.SPOON );
        try {
          KettleEnvironment.init();//重点
        } catch ( KettleException e ) {
          return e;
        }
        return null;
      }
    } );
}

KettleEnvironment.java中的init()方法

  public static void init( boolean simpleJndi ) throws KettleException {
    init( Arrays.asList(
      RowDistributionPluginType.getInstance(),
      StepPluginType.getInstance(),
      StepDialogFragmentType.getInstance(),
      PartitionerPluginType.getInstance(),
      JobEntryPluginType.getInstance(),
      JobEntryDialogFragmentType.getInstance(),
      LogTablePluginType.getInstance(),
      RepositoryPluginType.getInstance(),
      LifecyclePluginType.getInstance(),
      KettleLifecyclePluginType.getInstance(),
      ImportRulePluginType.getInstance(),
      CartePluginType.getInstance(),
      CompressionPluginType.getInstance(),
      AuthenticationProviderPluginType.getInstance(),
      AuthenticationConsumerPluginType.getInstance(),
      EnginePluginType.getInstance()
    ), simpleJndi );
  }

此方法中初始化了很多PluginTypeInterface类型对象。列如StepPluginType、JobEntryPluginType等多个插件类型

KettleEnvironment.java中另一个重载init()方法

public static void init( List<PluginTypeInterface> pluginClasses, boolean simpleJndi ) throws KettleException {
   KettleClientEnvironment.init();
}

KettleCleintEnvironment.java中的init()方法

 public static synchronized void init() throws KettleException {
    init( Arrays.asList( LoggingPluginType.getInstance(),
      ValueMetaPluginType.getInstance(),
      DatabasePluginType.getInstance(),
      ExtensionPointPluginType.getInstance(),
      TwoWayPasswordEncoderPluginType.getInstance() ) );
  }

此方法中又初始化了ValueMetaPluginType、DatabasePluginType等几个插件类型

KettleCleintEnvironment.java中的另一个init()重载方法

 public static synchronized void init( List<PluginTypeInterface> pluginsToLoad ) throws KettleException {    
    PluginRegistry.init();
  }

PluginRegistry.java中的init()

public static void init( boolean keepCache ) throws KettlePluginException {
registry.registerType( PluginRegistryPluginType.getInstance() );
for ( final PluginTypeInterface pluginType : pluginTypes ) {    
      registry.registerType( pluginType );//重点     
    }
}

PluginRegistry.java中的registerType()

  private void registerType( PluginTypeInterface pluginType ) throws KettlePluginException {
    registerPluginType( pluginType.getClass() );
    // Search plugins for this type...
    //
    long startScan = System.currentTimeMillis();
    pluginType.searchPlugins();
}

pluginType是PluginTypeInterface接口对象，而BasePluginType实现了PluginTypeInterface接口，只要实现了BasePluginType的类都会调用相应的接口实现。

PluginTypeInterface定义

public interface PluginTypeInterface {  
  void addObjectType( Class<?> clz, String xmlNodeName ); 
  String getId();
  String getName(); 
  List<PluginFolderInterface> getPluginFolders();  
  void searchPlugins() throws KettlePluginException; //重点 
  void handlePluginAnnotation( Class<?> clazz, java.lang.annotation.Annotation annotation,
    List<String> libraries, boolean nativePluginType, URL pluginFolder ) throws KettlePluginException;
  default boolean isFragment() {
    return false;
  }
}

BasePluginType实现

public abstract class BasePluginType implements PluginTypeInterface {
  public BasePluginType( Class<? extends java.lang.annotation.Annotation> pluginClass ) {
    this.pluginFolders = new ArrayList<>();
    this.log = new LogChannel( "Plugin type" );
    registry = PluginRegistry.getInstance();//重点
    this.pluginClass = pluginClass;
  }  

  //虽然getXmlPluginFile没有使用abstract,但有的插件重写了该方法（使用@override重写了）
  protected String getXmlPluginFile() {
    return null;
  }
  
  @Override
  public void searchPlugins() throws KettlePluginException {
    registerNatives();//内部插件
    registerPluginJars();//外部插件（以jar包形式加载）
    registerXmlPlugins();//外部插件（需配置plugin.xml文件进行加载）
  }

  protected void registerNatives() throws KettlePluginException {
     ...
     String xmlFile = getXmlPluginFile(); 
     ...
      InputStream inputStream = null;
    try {
      inputStream = getResAsStreamExternal( xmlFile );
      ...
      registerPlugins( inputStream );

    } catch ( KettleXMLException e ) { ...    
    } finally { ...  }
  } 
 
  //注意abstract为抽象方法，每个具体的插件必须实现自己registerXmlPlugins方法
  protected abstract void registerXmlPlugins() throws KettlePluginException; 

  @Override
  public List<PluginFolderInterface> getPluginFolders() {
    return pluginFolders;
  }

  protected PluginInterface registerPluginFromXmlResource( Node pluginNode, String path,
    Class<? extends PluginTypeInterface> pluginType, boolean nativePlugin, URL pluginFolder ) throws KettlePluginException {
    try {

      String idAttr = XMLHandler.getTagAttribute( pluginNode, "id" );
      String description = getTagOrAttribute( pluginNode, "description" );     
      String category = getTagOrAttribute( pluginNode, "category" );  
      ...
      PluginInterface pluginInterface =
        new Plugin(
          idAttr.split( "," ), pluginType, mainClassTypesAnnotation.value(), category, description, tooltip,
          iconFilename, false, nativePlugin, classMap, jarFiles, errorHelpFileFull, pluginFolder,
          documentationUrl, casesUrl, forumUrl, suggestion );
      registry.registerPlugin( pluginType, pluginInterface );//重点
      return pluginInterface;
    } catch ( Exception e ) {
      throw new KettlePluginException( BaseMessages.getString(
        PKG, "BasePluginType.RuntimeError.UnableToReadPluginXML.PLUGIN0001" ), e );
    }
  }

  

  protected void registerPluginJars() throws KettlePluginException {
    List<JarFileAnnotationPlugin> jarFilePlugins = findAnnotatedClassFiles( pluginClass.getName() );
    for ( JarFileAnnotationPlugin jarFilePlugin : jarFilePlugins ) {

      URLClassLoader urlClassLoader =
        createUrlClassLoader( jarFilePlugin.getJarFile(), getClass().getClassLoader() );

      try {
        Class<?> clazz = urlClassLoader.loadClass( jarFilePlugin.getClassName() );
        if ( clazz == null ) {
          throw new KettlePluginException( "Unable to load class: " + jarFilePlugin.getClassName() );
        }
        List<String> libraries = Arrays.stream( urlClassLoader.getURLs() )
          .map( URL::getFile )
          .collect( Collectors.toList() );
        Annotation annotation = clazz.getAnnotation( pluginClass );

        handlePluginAnnotation( clazz, annotation, libraries, false, jarFilePlugin.getPluginFolder() );
      } catch ( Exception e ) {
        // Ignore for now, don't know if it's even possible.
        LogChannel.GENERAL.logError(
          "Unexpected error registering jar plugin file: " + jarFilePlugin.getJarFile(), e );
      } finally {
        if ( urlClassLoader instanceof KettleURLClassLoader ) {
          ( (KettleURLClassLoader) urlClassLoader ).closeClassLoader();
        }
      }
    }
  }

 
  @Override
  public void handlePluginAnnotation( Class<?> clazz, java.lang.annotation.Annotation annotation,
    List<String> libraries, boolean nativePluginType, URL pluginFolder ) throws KettlePluginException {
    ...
    // Only one ID for now
    String[] ids = idList.split( "," );
    String packageName = extractI18nPackageName( annotation );
    String altPackageName = clazz.getPackage().getName();
    String pluginName = getTranslation( extractName( annotation ), packageName, 
    ....
    PluginInterface plugin =
      new Plugin(
        ids, this.getClass(), mainType.value(), category, pluginName, description, imageFile, separateClassLoader,
        classLoaderGroup, nativePluginType, classMap, libraries, null, pluginFolder, documentationUrl,
        casesUrl, forumUrl, suggestion );

    ParentFirst parentFirstAnnotation = clazz.getAnnotation( ParentFirst.class );
    if ( parentFirstAnnotation != null ) {
      registry.addParentClassLoaderPatterns( plugin, parentFirstAnnotation.patterns() );
    }
    registry.registerPlugin( this.getClass(), plugin );//重点
    ...    
  } 
}

Java知识要点：

（1）抽象方法为不具体的。继承抽象类的类需要实现所有的抽象方法

（2）含有抽象方法类必须使用abstract修饰

（3）抽象类中非抽象方法在继承类中可以重写

PluginRegistry.java中的registerPlugin()

 public void registerPlugin( Class<? extends PluginTypeInterface> pluginType, PluginInterface plugin )
      throws KettlePluginException {
    boolean changed = false; // Is this an add or an update?
    lock.writeLock().lock();
    try {
      if ( plugin.getIds()[0] == null ) {
        throw new KettlePluginException( "Not a valid id specified in plugin :" + plugin );
      }

      // Keep the list of plugins sorted by name...
      //
      Set<PluginInterface> list = pluginMap.computeIfAbsent( pluginType, k -> new TreeSet<>( Plugin.nullStringComparator ) );

      if ( !list.add( plugin ) ) {
        list.remove( plugin );
        list.add( plugin );
        changed = true;
      }

      if ( !Utils.isEmpty( plugin.getCategory() ) ) {
        // Keep categories sorted in the natural order here too!
        //
        categoryMap.computeIfAbsent( pluginType, k -> new TreeSet<>( getNaturalCategoriesOrderComparator( pluginType ) ) )
          .add( plugin.getCategory() );
      }
    } finally {
      lock.writeLock().unlock();
      Set<PluginTypeListener> listeners = this.listeners.get( pluginType );
      if ( listeners != null ) {
        for ( PluginTypeListener listener : listeners ) {
          // Changed or added?
          if ( changed ) {
            listener.pluginChanged( plugin );
          } else {
            listener.pluginAdded( plugin );
          }
        }
      }
      synchronized ( this ) {
        notifyAll();
      }
    }
  }

二、关于Kettle机制几点说明

（1）以StepPluginType为例说明

（1）StepPluginType重写了抽象类BasePluginType中getXmlPluginFile()方法返回的就是kettle-steps.xml文件名称而 getXmlPluginFile()方法是在BasePluginType的registerNatives() 中调用，从而不同插件公用了registerNatives()方法而实现了不同插件文件的解析。
（2）实现的BasePluginType中抽象的registerXmlPlugins()方法，实现了自己Plugins下外部插件文件解析逻辑

public class StepPluginType extends BasePluginType implements PluginTypeInterface {
  private static StepPluginType stepPluginType;
  protected StepPluginType() {
    super( Step.class, "STEP", "Step" );
    populateFolders( "steps" );
  }
  public static StepPluginType getInstance() {
    if ( stepPluginType == null ) {
      stepPluginType = new StepPluginType();
    }
    return stepPluginType;
  }
  @Override
  protected String getXmlPluginFile() {
    return Const.XML_FILE_KETTLE_STEPS;
  }
  protected void registerXmlPlugins() throws KettlePluginException {
    for ( PluginFolderInterface folder : pluginFolders ) {

      if ( folder.isPluginXmlFolder() ) {
        List<FileObject> pluginXmlFiles = findPluginXmlFiles( folder.getFolder() );
        for ( FileObject file : pluginXmlFiles ) {

          try {
            Document document = XMLHandler.loadXMLFile( file );
            Node pluginNode = XMLHandler.getSubNode( document, "plugin" );
            if ( pluginNode != null ) {
              registerPluginFromXmlResource( pluginNode, KettleVFS.getFilename( file.getParent() ), this
                .getClass(), false, file.getParent().getURL() );
            }
          } catch ( Exception e ) {
            // We want to report this plugin.xml error, perhaps an XML typo or something like that...
            //
            log.logError( "Error found while reading step plugin.xml file: " + file.getName().toString(), e );
          }
        }
      }
    }
  }  
}

（3）kettle-steps.xml文件格式（节选）

<?xml version="1.0" encoding="UTF-8"?>
<steps> 
  <step id="TextFileInput">
    <description>i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.TextFileInput</description>
    <classname>org.pentaho.di.trans.steps.fileinput.text.TextFileInputMeta</classname>
    <category>i18n:org.pentaho.di.trans.step:BaseStep.Category.Input</category>
    <tooltip>i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.TextInputFile</tooltip>
    <iconfile>ui/images/TFI.svg</iconfile>
    <documentation_url>Products/Text_File_Input</documentation_url>
    <cases_url>https://jira.pentaho.com/browse/PDI</cases_url>
    <forum_url>https://community.hds.com/community/products-and-solutions/pentaho/data-integration</forum_url>
  </step> 
  <step id="ExecSQL">
    <description>i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.ExcuteSQL</description>
    <classname>org.pentaho.di.trans.steps.sql.ExecSQLMeta</classname>
    <category>i18n:org.pentaho.di.trans.step:BaseStep.Category.Scripting</category>
    <tooltip>i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.ExecuteSQL</tooltip>
    <iconfile>ui/images/SQL.svg</iconfile>
    <documentation_url>Products/Execute_SQL_Script</documentation_url>
    <cases_url/>
    <forum_url/>
  </step>   
</steps>

（2）关于外部插件以jar包方式加装的说明

 protected void registerPluginJars() throws KettlePluginException {
    List<JarFileAnnotationPlugin> jarFilePlugins = findAnnotatedClassFiles( pluginClass.getName() );
    for ( JarFileAnnotationPlugin jarFilePlugin : jarFilePlugins ) {

      URLClassLoader urlClassLoader =
        createUrlClassLoader( jarFilePlugin.getJarFile(), getClass().getClassLoader() );

      try {
        Class<?> clazz = urlClassLoader.loadClass( jarFilePlugin.getClassName() );
        if ( clazz == null ) {
          throw new KettlePluginException( "Unable to load class: " + jarFilePlugin.getClassName() );
        }
        List<String> libraries = Arrays.stream( urlClassLoader.getURLs() )
          .map( URL::getFile )
          .collect( Collectors.toList() );
        Annotation annotation = clazz.getAnnotation( pluginClass );

        handlePluginAnnotation( clazz, annotation, libraries, false, jarFilePlugin.getPluginFolder() );
      } catch ( Exception e ) {
        // Ignore for now, don't know if it's even possible.
        LogChannel.GENERAL.logError(
          "Unexpected error registering jar plugin file: " + jarFilePlugin.getJarFile(), e );
      } finally {
        if ( urlClassLoader instanceof KettleURLClassLoader ) {
          ( (KettleURLClassLoader) urlClassLoader ).closeClassLoader();
        }
      }
    }
  }

调试分析现有源码目录不能正确加载jar包形式的外部插件，需要将根目录下的plugins下插件jar包拷贝到/ui/plugins目录下，外部插件才可以正常加载。

三、Kettle左边资源树面板的创建

Spoon.java中的refreshCoreObjects()

  public void refreshCoreObjects() {
   //Trans节点
    if ( showTrans ) {
  
      PluginRegistry registry = PluginRegistry.getInstance();
      //获取所有Trans插件
      final List<PluginInterface> baseSteps = registry.getPlugins( StepPluginType.class );
     //获取所有Trans插件分类
      final List<String> baseCategories = registry.getCategories( StepPluginType.class );
     
      for ( String baseCategory : baseCategories ) {
        TreeItem item = new TreeItem( coreObjectsTree, SWT.NONE );
        item.setText( baseCategory );
        item.setImage( GUIResource.getInstance().getImageFolder() );
    //Job节点   
    if ( showJob ) {
   
      PluginRegistry registry = PluginRegistry.getInstance();
        //获取所有Job插件
      List<PluginInterface> baseJobEntries = registry.getPlugins( JobEntryPluginType.class );
    //获取所有Job插件分类
      List<String> baseCategories = registry.getCategories( JobEntryPluginType.class );
      TreeItem generalItem = null;
      for ( String baseCategory : baseCategories ) {
        TreeItem item = new TreeItem( coreObjectsTree, SWT.NONE );
        item.setText( baseCategory );
        item.setImage( GUIResource.getInstance().getImageFolder() );       
 

主要使用 registry.getPlugins和registry.getCategories创建Trans节点的分类节点和分类下的插件节点。Job节点类似Trans的创建。Trans和Job节点创建不同在于一个传递StepPluginType.class与JobEntryPluginType.class