zenoh 动态plugin 的设计简单说明

zenoh 提供了一种插件化的架构设计，对于storage plugin 可以灵活的扩展（当然其他模式也是支持的，比如rest，mqtt。。。。）以下是一个简单说明（zenoh 目前是包含了静态插件以及动态插件，主要说明动态插件）

技术实现机制

技术上zenoh 对于插件的支持是基于了libloading，zenoh 实现了一个插件管理器，通过读取配置文件，然后进行插件的加载

pub(crate) fn load_plugins(config: &Config) -> PluginsManager {
    // 动态插件加载器
    let mut manager = PluginsManager::dynamic(config.libloader(), PLUGIN_PREFIX.to_string());
    // Static plugins are to be added here, with `.add_static::<PluginType>()`
    for plugin_load in config.plugins().load_requests() {
        let PluginLoad {
            id,
            name,
            paths,
            required,
        } = plugin_load;
        tracing::info!(
            "Loading {req} plugin \"{id}\"",
            req = if required { "required" } else { "" }
        );
        // 集合
        if let Err(e) = load_plugin(&mut manager, &name, &id, &paths, required) {
            if required {
                panic!("Plugin load failure: {}", e)
            } else {
                tracing::error!("Plugin load failure: {}", e)
            }
        }
    }
    manager
}

插件加载

pub(crate) fn load_plugin(
    plugin_mgr: &mut PluginsManager,
    name: &str,
    id: &str,
    paths: &Option<Vec<String>>,
    required: bool,
) -> ZResult<()> {
    let declared = if let Some(declared) = plugin_mgr.plugin_mut(name) {
        tracing::warn!("Plugin `{}` was already declared", declared.id());
        declared
    } else if let Some(paths) = paths {
        plugin_mgr.declare_dynamic_plugin_by_paths(name, id, paths, required)?
    } else {
        plugin_mgr.declare_dynamic_plugin_by_name(id, name, required)?
    };

    if let Some(loaded) = declared.loaded_mut() {
        tracing::warn!(
            "Plugin `{}` was already loaded from {}",
            loaded.id(),
            loaded.path()
        );
    } else {
        let _ = declared.load()?;
    };
    Ok(())
}

plugin trait 定义

pub trait Plugin: Sized + 'static {
    type StartArgs: PluginStartArgs;
    type Instance: PluginInstance;
    /// Plugins' default name when statically linked.
    const DEFAULT_NAME: &'static str;
    /// Plugin's version. Used only for information purposes. It's recommended to use [plugin_version!](crate::plugin_version!) macro to generate this string.
    const PLUGIN_VERSION: &'static str;
    /// Plugin's long version (with git commit hash). Used only for information purposes. It's recommended to use [plugin_version!](crate::plugin_version!) macro to generate this string.
    const PLUGIN_LONG_VERSION: &'static str;
    /// Starts your plugin. Use `Ok` to return your plugin's control structure
    fn start(name: &str, args: &Self::StartArgs) -> ZResult<Self::Instance>;
}

DynamicPluginStarter动态插件的starter 定义（实际上就是包含了启动插件以及一些参数，具体由DynamicPlugin 去调用start方法启动，DynamicPlugin 是）

struct DynamicPluginStarter<StartArgs, Instance> {
    _lib: Library,
    path: PathBuf,
    vtable: PluginVTable<StartArgs, Instance>,
}

impl<StartArgs: PluginStartArgs, Instance: PluginInstance>
    DynamicPluginStarter<StartArgs, Instance>
{
    fn get_vtable(lib: &Library, path: &Path) -> ZResult<PluginVTable<StartArgs, Instance>> {
        tracing::debug!("Loading plugin {}", path.to_str().unwrap(),);
        // 获取 declare_plugin macro 暴露的方法，属于libloading 包提供的能力
        let get_plugin_loader_version =
            unsafe { lib.get::<fn() -> PluginLoaderVersion>(b"get_plugin_loader_version")? };
        let plugin_loader_version = get_plugin_loader_version();
        tracing::debug!("Plugin loader version: {}", &plugin_loader_version);
        if plugin_loader_version != PLUGIN_LOADER_VERSION {
            bail!(
                "Plugin loader version mismatch: host = {}, plugin = {}",
                PLUGIN_LOADER_VERSION,
                plugin_loader_version
            );
        }
        // 获取 declare_plugin macro 暴露的方法，属于libloading 包提供的能力
        let get_compatibility = unsafe { lib.get::<fn() -> Compatibility>(b"get_compatibility")? };
        let mut plugin_compatibility_record = get_compatibility();
        let mut host_compatibility_record =
            Compatibility::with_empty_plugin_version::<StartArgs, Instance>();
        tracing::debug!(
            "Plugin compatibility record: {:?}",
            &plugin_compatibility_record
        );
        if !plugin_compatibility_record.compare(&mut host_compatibility_record) {
            bail!(
                "Plugin compatibility mismatch:\nHost:\n{}Plugin:\n{}",
                host_compatibility_record,
                plugin_compatibility_record
            );
        }
         // 获取 declare_plugin macro 暴露的方法，属于libloading 包提供的能力
        let load_plugin =
            unsafe { lib.get::<fn() -> PluginVTable<StartArgs, Instance>>(b"load_plugin")? };

        Ok(load_plugin())
    }
    // 创建插件，通过获取vtalbe （vtalbe 实际上存储了插件的版本信息，以及启动参数，为了暴露相关方法，zenoh 实现了一个macro 方法插件加载的时候获取信息）
    fn new(lib: Library, path: PathBuf) -> ZResult<Self> {
        let vtable = Self::get_vtable(&lib, &path)
            .map_err(|e| format!("Error loading {}: {}", path.to_str().unwrap(), e))?;
        Ok(Self {
            _lib: lib,
            path,
            vtable,
        })
    }
    // 插件启动
    fn start(&self, name: &str, args: &StartArgs) -> ZResult<Instance> {
        (self.vtable.start)(name, args)
    }
    fn path(&self) -> &str {
        self.path.to_str().unwrap()
    }
}

declare_plugin plugin 信息暴露

#[macro_export]
macro_rules! declare_plugin {
    ($ty: path) => {
        #[no_mangle]
        fn get_plugin_loader_version() -> $crate::PluginLoaderVersion {
            $crate::PLUGIN_LOADER_VERSION
        }

        #[no_mangle]
        fn get_compatibility() -> $crate::Compatibility {
            $crate::Compatibility::with_plugin_version::<
                <$ty as $crate::Plugin>::StartArgs,
                <$ty as $crate::Plugin>::Instance,
                $ty,
            >()
        }

        #[no_mangle]
        fn load_plugin() -> $crate::PluginVTable<
            <$ty as $crate::Plugin>::StartArgs,
            <$ty as $crate::Plugin>::Instance,
        > {
            $crate::PluginVTable::new::<$ty>()
        }
    };
}

PluginVTable struct (c 布局)

#[repr(C)]
pub struct PluginVTable<StartArgs, Instance> {
    pub plugin_version: &'static str,
    pub plugin_long_version: &'static str,
    pub start: StartFn<StartArgs, Instance>,
}

插件的启动：zenoh 代码中使用了不少feature 机制，对于plugin 的支持也是一个feature，对于开启了特性的才会使用,具体处理是在RuntimeBuilder 的builder 方法中

 #[cfg(feature = "plugins")]
        let plugins_manager = plugins_manager
            .take()
            .unwrap_or_else(|| load_plugins(&config));
        // Admin space creation flag
        let start_admin_space = *config.adminspace.enabled();
        // SHM lazy init flag
        #[cfg(feature = "shared-memory")]
        let shm_init_mode = *config.transport.shared_memory.mode();

        let config = Notifier::new(crate::config::Config(config));
        let runtime = Runtime {
            state: Arc::new(RuntimeState {
                zid: zid.into(),
                whatami,
                next_id: AtomicU32::new(1), // 0 is reserved for routing core
                router,
                config: config.clone(),
                manager: transport_manager,
                transport_handlers: std::sync::RwLock::new(vec![]),
                locators: std::sync::RwLock::new(vec![]),
                hlc,
                task_controller: TaskController::default(),
                #[cfg(feature = "plugins")]
                plugins_manager: Mutex::new(plugins_manager),
                start_conditions: Arc::new(StartConditions::default()),
                pending_connections: tokio::sync::Mutex::new(HashSet::new()),
            }),
        };
        *handler.runtime.write().unwrap() = Runtime::downgrade(&runtime);
        get_mut_unchecked(&mut runtime.state.router.clone()).init_link_state(runtime.clone())?;

        // Admin space
        if start_admin_space {
            AdminSpace::start(&runtime, LONG_VERSION.clone()).await;
        }

        // Start plugins 启动插件
        #[cfg(feature = "plugins")]
        start_plugins(&runtime);

start_plugins 处理, 实际上就结合插件的配置参数，以及插件管理器获取到的插件调用start 方法启动

pub(crate) fn start_plugins(runtime: &Runtime) {
    let mut manager = runtime.plugins_manager();
    for plugin in manager.loaded_plugins_iter_mut() {
        let required = plugin.required();
        tracing::info!(
            "Starting {req} plugin \"{name}\"",
            req = if required { "required" } else { "" },
            name = plugin.id()
        );
        match plugin.start(runtime) {
            Ok(_) => {
                tracing::info!(
                    "Successfully started plugin {} from {:?}",
                    plugin.id(),
                    plugin.path()
                );
            }
            Err(e) => {
                let report = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| e.to_string())) {
                        Ok(s) => s,
                        Err(_) => panic!("Formatting the error from plugin {} ({:?}) failed, this is likely due to ABI unstability.\r\nMake sure your plugin was built with the same version of cargo as zenohd", plugin.name(), plugin.path()),
                    };
                if required {
                    panic!(
                        "Plugin \"{}\" failed to start: {}",
                        plugin.id(),
                        if report.is_empty() {
                            "no details provided"
                        } else {
                            report.as_str()
                        }
                    );
                } else {
                    tracing::error!(
                        "Required plugin \"{}\" failed to start: {}",
                        plugin.id(),
                        if report.is_empty() {
                            "no details provided"
                        } else {
                            report.as_str()
                        }
                    );
                }
            }
        }
        tracing::info!("Finished loading plugins");
    }
}

说明

zenoh 的动态plugin 机制核心还是利用了libloading 的能力实现的，毕竟libloading 提供的能力比较方法， 基于ffi机制实际我们也可以实现的，就是相对复杂一些

参考资料

https://github.com/nagisa/rust_libloading

https://github.com/eclipse-zenoh/zenoh/tree/main/plugins

https://docs.rs/libloading/latest/libloading/struct.Library.html#method.new