Pre-cache lang/en files in RAM in miniserv.pl

diff --git a/CHANGELOG b/CHANGELOG
index fdbd212..1b57157 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -134,3 +134,5 @@ Dutch translation updates, thanks to Gandyman.
 Polish translation updates, thanks to Dariusz Dêbowski.
 ---- Changes since 1.520 ----
 Catalan translation updates by Jaume Badiella.
+---- Changes since 1.530 ----
+Sped up the loading of language files by pre-caching them in memory when Webmin is started, and not performing sub-string substitutions in most modules.

diff --git a/WebminCore.pm b/WebminCore.pm
index cde1607..e9fb863 100644 (file)
--- a/WebminCore.pm
+++ b/WebminCore.pm
@@ -20,7 +20,7 @@ require Exporter;
 # Add functions in web-lib-funcs.pl
 # Generated with :
 # grep -h "^sub " web-lib-funcs.pl ui-lib.pl | sed -e 's/sub /\&/' | xargs echo
-@EXPORT = qw(&read_file &read_file_cached &write_file &html_escape &quote_escape &tempname &transname &trunc &indexof &indexoflc &sysprint &check_ipaddress &check_ip6address &generate_icon &urlize &un_urlize &include &copydata &ReadParseMime &ReadParse &read_fully &read_parse_mime_callback &read_parse_mime_javascript &PrintHeader &header &get_html_title &get_html_framed_title &get_html_status_line &popup_header &footer &popup_footer &load_theme_library &redirect &kill_byname &kill_byname_logged &find_byname &error &popup_error &error_setup &wait_for &fast_wait_for &has_command &make_date &file_chooser_button &popup_window_button &read_acl &acl_filename &acl_check &get_miniserv_config &put_miniserv_config &restart_miniserv &reload_miniserv &check_os_support &http_download &complete_http_download &ftp_download &ftp_upload &no_proxy &open_socket &download_timeout &ftp_command &to_ipaddress &to_ip6address &to_hostname &icons_table &replace_file_line &read_file_lines &flush_file_lines &unflush_file_lines &unix_user_input &unix_group_input &hlink &user_chooser_button &group_chooser_button &foreign_check &foreign_exists &foreign_available &foreign_require &foreign_call &foreign_config &foreign_installed &foreign_defined &get_system_hostname &get_webmin_version &get_module_acl &get_group_module_acl &save_module_acl &save_group_module_acl &init_config &load_language &text_subs &text &encode_base64 &decode_base64 &get_module_info &get_all_module_infos &get_theme_info &list_languages &read_env_file &write_env_file &lock_file &unlock_file &test_lock &unlock_all_files &can_lock_file &webmin_log &additional_log &webmin_debug_log &system_logged &backquote_logged &backquote_with_timeout &backquote_command &kill_logged &rename_logged &rename_file &symlink_logged &symlink_file &link_file &make_dir &set_ownership_permissions &unlink_logged &unlink_file &copy_source_dest &remote_session_name &remote_foreign_require &remote_foreign_call &remote_foreign_check &remote_foreign_config &remote_eval &remote_write &remote_read &remote_finished &remote_error_setup &remote_rpc_call &remote_multi_callback &remote_multi_callback_error &serialise_variable &unserialise_variable &other_groups &date_chooser_button &help_file &seed_random &disk_usage_kb &recursive_disk_usage &help_search_link &make_http_connection &read_http_connection &write_http_connection &close_http_connection &clean_environment &reset_environment &progress_callback &switch_to_remote_user &switch_to_unix_user &create_user_config_dirs &create_missing_homedir &filter_javascript &resolve_links &simplify_path &same_file &flush_webmin_caches &list_usermods &available_usermods &get_available_module_infos &get_visible_module_infos &get_visible_modules_categories &is_under_directory &parse_http_url &check_clicks_function &load_entities_map &entities_to_ascii &get_product_name &get_charset &get_display_hostname &save_module_config &save_user_module_config &nice_size &get_perl_path &get_goto_module &select_all_link &select_invert_link &select_rows_link &check_pid_file &get_mod_lib &module_root_directory &list_mime_types &guess_mime_type &open_tempfile &close_tempfile &print_tempfile &is_selinux_enabled &get_clear_file_attributes &reset_file_attributes &cleanup_tempnames &open_lock_tempfile &END &month_to_number &number_to_month &get_rbac_module_acl &supports_rbac &supports_ipv6 &use_rbac_module_acl &execute_command &open_readfile &open_execute_command &translate_filename &translate_command &register_filename_callback &register_command_callback &capture_function_output &capture_function_output_tempfile &modules_chooser_button &substitute_template &running_in_zone &running_in_vserver &running_in_xen &running_in_openvz &list_categories &is_readonly_mode &command_as_user &list_osdn_mirrors &convert_osdn_url &get_current_dir &supports_users &supports_symlinks &quote_path &get_windows_root &read_file_contents &unix_crypt &split_quoted_string &write_to_http_cache &check_in_http_cache &supports_javascript &ui_table_start &ui_table_end &ui_table_row &ui_table_hr &ui_table_span &ui_columns_start &ui_columns_row &ui_columns_header &ui_checked_columns_row &ui_radio_columns_row &ui_columns_end &ui_columns_table &ui_form_columns_table &ui_form_start &ui_form_end &ui_textbox &ui_filebox &ui_bytesbox &ui_upload &ui_password &ui_hidden &ui_select &ui_multi_select &ui_multi_select_javascript &ui_radio &ui_yesno_radio &ui_checkbox &ui_oneradio &ui_textarea &ui_user_textbox &ui_group_textbox &ui_opt_textbox &ui_submit &ui_reset &ui_button &ui_date_input &ui_buttons_start &ui_buttons_end &ui_buttons_row &ui_buttons_hr &ui_post_header &ui_pre_footer &ui_print_header &ui_print_unbuffered_header &ui_print_footer &ui_config_link &ui_print_endpage &ui_subheading &ui_links_row &ui_hidden_javascript &ui_hidden_start &ui_hidden_end &ui_hidden_table_row_start &ui_hidden_table_row_end &ui_hidden_table_start &ui_hidden_table_end &ui_tabs_start &ui_tabs_end &ui_tabs_start_tab &ui_tabs_start_tabletab &ui_tabs_end_tab &ui_tabs_end_tabletab &ui_max_text_width &ui_radio_selector &ui_radio_selector_javascript &ui_grid_table &ui_radio_table &ui_up_down_arrows &ui_hr &ui_nav_link &ui_confirmation_form &js_disable_inputs &ui_page_flipper &js_checkbox_disable &js_redirect &get_module_name clear_time_locale reset_time_locale eval_as_unix_user get_userdb_string connect_userdb disconnect_userdb split_userdb_string);
+@EXPORT = qw(&read_file &read_file_cached &read_file_cached_with_stat &write_file &html_escape &quote_escape &tempname &transname &trunc &indexof &indexoflc &sysprint &check_ipaddress &check_ip6address &generate_icon &urlize &un_urlize &include &copydata &ReadParseMime &ReadParse &read_fully &read_parse_mime_callback &read_parse_mime_javascript &PrintHeader &header &get_html_title &get_html_framed_title &get_html_status_line &popup_header &footer &popup_footer &load_theme_library &redirect &kill_byname &kill_byname_logged &find_byname &error &popup_error &error_setup &wait_for &fast_wait_for &has_command &make_date &file_chooser_button &popup_window_button &read_acl &acl_filename &acl_check &get_miniserv_config &put_miniserv_config &restart_miniserv &reload_miniserv &check_os_support &http_download &complete_http_download &ftp_download &ftp_upload &no_proxy &open_socket &download_timeout &ftp_command &to_ipaddress &to_ip6address &to_hostname &icons_table &replace_file_line &read_file_lines &flush_file_lines &unflush_file_lines &unix_user_input &unix_group_input &hlink &user_chooser_button &group_chooser_button &foreign_check &foreign_exists &foreign_available &foreign_require &foreign_call &foreign_config &foreign_installed &foreign_defined &get_system_hostname &get_webmin_version &get_module_acl &get_group_module_acl &save_module_acl &save_group_module_acl &init_config &load_language &text_subs &text &encode_base64 &decode_base64 &get_module_info &get_all_module_infos &get_theme_info &list_languages &read_env_file &write_env_file &lock_file &unlock_file &test_lock &unlock_all_files &can_lock_file &webmin_log &additional_log &webmin_debug_log &system_logged &backquote_logged &backquote_with_timeout &backquote_command &kill_logged &rename_logged &rename_file &symlink_logged &symlink_file &link_file &make_dir &set_ownership_permissions &unlink_logged &unlink_file &copy_source_dest &remote_session_name &remote_foreign_require &remote_foreign_call &remote_foreign_check &remote_foreign_config &remote_eval &remote_write &remote_read &remote_finished &remote_error_setup &remote_rpc_call &remote_multi_callback &remote_multi_callback_error &serialise_variable &unserialise_variable &other_groups &date_chooser_button &help_file &seed_random &disk_usage_kb &recursive_disk_usage &help_search_link &make_http_connection &read_http_connection &write_http_connection &close_http_connection &clean_environment &reset_environment &progress_callback &switch_to_remote_user &switch_to_unix_user &create_user_config_dirs &create_missing_homedir &filter_javascript &resolve_links &simplify_path &same_file &flush_webmin_caches &list_usermods &available_usermods &get_available_module_infos &get_visible_module_infos &get_visible_modules_categories &is_under_directory &parse_http_url &check_clicks_function &load_entities_map &entities_to_ascii &get_product_name &get_charset &get_display_hostname &save_module_config &save_user_module_config &nice_size &get_perl_path &get_goto_module &select_all_link &select_invert_link &select_rows_link &check_pid_file &get_mod_lib &module_root_directory &list_mime_types &guess_mime_type &open_tempfile &close_tempfile &print_tempfile &is_selinux_enabled &get_clear_file_attributes &reset_file_attributes &cleanup_tempnames &open_lock_tempfile &END &month_to_number &number_to_month &get_rbac_module_acl &supports_rbac &supports_ipv6 &use_rbac_module_acl &execute_command &open_readfile &open_execute_command &translate_filename &translate_command &register_filename_callback &register_command_callback &capture_function_output &capture_function_output_tempfile &modules_chooser_button &substitute_template &running_in_zone &running_in_vserver &running_in_xen &running_in_openvz &list_categories &is_readonly_mode &command_as_user &list_osdn_mirrors &convert_osdn_url &get_current_dir &supports_users &supports_symlinks &quote_path &get_windows_root &read_file_contents &unix_crypt &split_quoted_string &write_to_http_cache &check_in_http_cache &supports_javascript &ui_table_start &ui_table_end &ui_table_row &ui_table_hr &ui_table_span &ui_columns_start &ui_columns_row &ui_columns_header &ui_checked_columns_row &ui_radio_columns_row &ui_columns_end &ui_columns_table &ui_form_columns_table &ui_form_start &ui_form_end &ui_textbox &ui_filebox &ui_bytesbox &ui_upload &ui_password &ui_hidden &ui_select &ui_multi_select &ui_multi_select_javascript &ui_radio &ui_yesno_radio &ui_checkbox &ui_oneradio &ui_textarea &ui_user_textbox &ui_group_textbox &ui_opt_textbox &ui_submit &ui_reset &ui_button &ui_date_input &ui_buttons_start &ui_buttons_end &ui_buttons_row &ui_buttons_hr &ui_post_header &ui_pre_footer &ui_print_header &ui_print_unbuffered_header &ui_print_footer &ui_config_link &ui_print_endpage &ui_subheading &ui_links_row &ui_hidden_javascript &ui_hidden_start &ui_hidden_end &ui_hidden_table_row_start &ui_hidden_table_row_end &ui_hidden_table_start &ui_hidden_table_end &ui_tabs_start &ui_tabs_end &ui_tabs_start_tab &ui_tabs_start_tabletab &ui_tabs_end_tab &ui_tabs_end_tabletab &ui_max_text_width &ui_radio_selector &ui_radio_selector_javascript &ui_grid_table &ui_radio_table &ui_up_down_arrows &ui_hr &ui_nav_link &ui_confirmation_form &js_disable_inputs &ui_page_flipper &js_checkbox_disable &js_redirect &get_module_name clear_time_locale reset_time_locale eval_as_unix_user get_userdb_string connect_userdb disconnect_userdb split_userdb_string);
 
 # Add global variables in web-lib.pl
 push(@EXPORT, qw(&unique));

diff --git a/miniserv.pl b/miniserv.pl
index 0d693ae..9aec616 100755 (executable)
--- a/miniserv.pl
+++ b/miniserv.pl
@@ -376,6 +376,9 @@ if ($config{'debuglog'}) {
 %webmincron_last = ( );
 &read_file($config{'webmincron_last'}, \%webmincron_last);
 
+# Pre-cache lang files
+&precache_files();
+
 if ($config{'inetd'}) {
        # We are being run from inetd - go direct to handling the request
        &redirect_stderr_to_log();
@@ -4156,6 +4159,7 @@ sub reload_config_file
 &read_mime_types();
 &build_config_mappings();
 &read_webmin_crons();
+&precache_files();
 if ($config{'session'}) {
        dbmclose(%sessiondb);
        dbmopen(%sessiondb, $config{'sessiondb'}, 0700);
@@ -4206,6 +4210,7 @@ my %vital = ("port", 80,
          "max_post", 10000,
          "expires", 7*24*60*60,
          "pam_test_user", "root",
+         "precache", "lang/en */lang/en",
         );
 foreach my $v (keys %vital) {
        if (!$config{$v}) {
@@ -5635,6 +5640,22 @@ foreach my $f (readdir(CRONS)) {
        }
 }
 
+# precache_files()
+# Read into the Webmin cache all files marked for pre-caching
+sub precache_files
+{
+undef(%main::read_file_cache);
+foreach my $g (split(/\s+/, $config{'precache'})) {
+       foreach my $f (glob("$config{'root'}/$g")) {
+               my @st = stat($f);
+               next if (!@st);
+               $main::read_file_cache{$f} = { };
+               &read_file($f, $main::read_file_cache{$f});
+               $main::read_file_cache_time{$f} = $st[9];
+               }
+       }
+}
+
 # Check if some address is valid IPv4, returns 1 if so.
 sub check_ipaddress
 {

diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl
index 9b28b0c..8561e06 100755 (executable)
--- a/web-lib-funcs.pl
+++ b/web-lib-funcs.pl
@@ -105,6 +105,26 @@ else {
                }
        }
 }
+
+=head2 read_file_cached_with_stat(file, &hash, [&order], [lowercase], [split-char])
+
+Like read_file, but reads from an in-memory cache if the file has already been
+read in this Webmin script AND has not changed.
+
+=cut
+sub read_file_cached_with_stat
+{
+my $realfile = &translate_filename($_[0]);
+my $t = $main::read_file_cache_time{$realfile};
+my @st = stat($realfile);
+if ($t && $st[9] != $t) {
+       # Changed, invalidate cache
+       delete($main::read_file_cache{$realfile});
+       }
+my $rv = &read_file_cached(@_);
+$main::read_file_cache_time{$realfile} = $st[9];
+return $rv;
+}
  
 =head2 write_file(file, &hash, [join-char])
 
@@ -4414,7 +4434,7 @@ my ($dir) = ($_[1] || "lang");
 
 # Read global lang files
 foreach my $o (@lang_order_list) {
-       my $ok = &read_file_cached("$root/$dir/$o", \%text);
+       my $ok = &read_file_cached_with_stat("$root/$dir/$o", \%text);
        return () if (!$ok && $o eq $default_lang);
        }
 if ($ol) {
@@ -4430,10 +4450,10 @@ if ($_[0]) {
        delete($text{'__norefs'});
        my $mdir = &module_root_directory($_[0]);
        foreach my $o (@lang_order_list) {
-               &read_file_cached("$mdir/$dir/$o", \%text);
+               &read_file_cached_with_stat("$mdir/$dir/$o", \%text);
                }
        if ($ol) {
-               foreach $o (@lang_order_list) {
+               foreach my $o (@lang_order_list) {
                        &read_file_cached("$mdir/$ol/$o", \%text);
                        }
                }