mclapply 与 lme4 和长向量

Question

我正在使用 parallel 包中的 mclapply 在高性能集群上使用 lme4 包来估计混合 glmer 模型。我遇到了问题 described here. I apply the suggested fix of adding mc.preschedule=F, but the problem persists. The code is set up as described here。

我不确定如何绕过它，有什么想法吗？我应该切换到另一种并行化方法吗？如果可以，怎么做？

这是我的代码，但基本上它遵循链接文章的逻辑：

rm(list = ls())

require(lme4)
require(parallel)

load(file="//share//home//eborbath//ess_rescaled.Rda") # load data

# paralelizing function

f_lmer_mc = function(data, calls, mc.cores) {
  require(parallel)
  if (is.data.frame(data)) 
    data = replicate(length(calls), data, simplify = F)
  for (i in 1:length(data)) attr(data[[i]], "cll") = calls[i]
  m.list = mclapply(data, function(i) eval(parse(text = attr(i, "cll"))), 
                    mc.cores = mc.cores, mc.preschedule = FALSE)
  return(m.list)
}

##########
# Models #
##########


controls <- c("gender", "agea", "eduyrs", "domicil", "unemployed", "rideol", "union", "pid", "hincfel")
values <- c("conformity", "universalism", "security")
issues <- c("gincdif", "freehms")
agr.ctrl <- c("gdp_wb_ppp", "wb_vae")
lr.agr <- c("lr_rsquar_std", "ri_l2_std")
val.agr <- c("mean_univ", "mean_conf", "mean_secur")
end <- "1 + (1|cntry/countryyear), data=i, control=glmerControl(optimizer='bobyqa', optCtrl = list(maxfun = 1e9)), family=binomial(link='logit'))"

models = c(paste0("glmer(protest ~", paste(c(controls, end), collapse="+")),
paste0("glmer(protest ~", paste(c(controls, values, end), collapse="+")),
paste0("glmer(protest ~", paste(c(controls, values, issues, end), collapse="+")),
paste0("glmer(protest ~ region+", paste(c(controls, values, issues, end), collapse="+")),
paste0("glmer(protest ~ region+", paste(c(controls, values, issues, agr.ctrl, end), collapse="+")), 
paste0("glmer(protest ~ region+", paste(c(controls, values, issues, agr.ctrl, lr.agr, end), collapse="+")),
paste0("glmer(protest ~ region+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")), # until here it's only main effects
paste0("glmer(protest ~ region*rideol + region+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")), 
paste0("glmer(protest ~ region*rideol*year + region+year+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")), 
paste0("glmer(protest ~ region*rideol*year_num + region+year_num+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")), 
paste0("glmer(protest ~ region*soc_pop_eleches + region+soc_pop_eleches+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")), # now come the expl. models
paste0("glmer(protest ~ region*rideol*soc_pop_eleches + region+soc_pop_eleches+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*ri_l2_std + region+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*ri_l2_std*rideol + region+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*lr_rsquar_std + region+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*lr_rsquar_std*rideol + region+", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region+gov_genlr", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*gov_genlr + region+gov_genlr", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*gov_genlr*rideol + region+gov_genlr", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region+pol_lrecon", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region+pol_galtan", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region+pol_galtan+pol_lrecon", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*pol_lrecon+region+pol_galtan+pol_lrecon", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*pol_galtan+region+pol_galtan+pol_lrecon", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*pol_lrecon*rideol+region+pol_galtan+pol_lrecon", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")),
paste0("glmer(protest ~ region*pol_galtan*rideol+region+pol_galtan+pol_lrecon", paste(c(controls, values, issues, agr.ctrl, lr.agr, val.agr, end), collapse="+")))

m.list = f_lmer_mc(data, models, 24)

m.1 <- c(m.list[1:3])
m.2 <- c(m.list[4:6])
m.3 <- c(m.list[7:9])
m.4 <- c(m.list[10:12])
m.5 <- c(m.list[13:15])
m.6 <- c(m.list[16:18])
m.7 <- c(m.list[19:21])
m.8 <- c(m.list[22:24])
m.9 <- c(m.list[25:26])

save(m.1, data, file='m_1.RData')
save(m.2, data, file='m_2.RData')
save(m.3, data, file='m_3.RData')
save(m.4, data, file='m_4.RData')
save(m.5, data, file='m_5.RData')
save(m.6, data, file='m_6.RData')
save(m.7, data, file='m_7.RData')
save(m.8, data, file='m_8.RData')
save(m.9, data, file='m_9.RData')

这是相关的错误信息：

Error in sendMaster(try(eval(expr, env), silent = TRUE)) : 
  long vectors not supported yet: fork.c:378
Calls: f_lmer_mc ... mclapply -> lapply -> FUN -> mcparallel -> sendMaster

谢谢！

更新：

数据是公开可用数据的清理版本 European Social Survey. You can download the file from here (1.8 MB)

Answer 1

我认为发生此错误是因为分叉工作进程在序列化非常大的结果对象时出错。我已经能够使用以下代码在 R 3.3.2 中重现此错误：

library(parallel)
r <- mclapply(1:2, function(i) 1:2^30, mc.cores=2, mc.preschedule=FALSE)

然而，这个例子对我来说适用于使用 64 位版本的 R 3.4.3，因此序列化限制似乎在更高版本的 R 中已经被删除（或至少增加）。

我建议您要么尝试将结果对象的大小减小到 2GB 以下，要么使用最新版本的 R。

Answer 2

扩展我上面的评论：

I see you're replicating the data set and then sending it to all the processes. I haven't done parallel stuff in a while, but you might not need to do that; the vignette says "with mclapply all the packages and objects we use are automatically available on the workers." If so, that would take care of going to the processes, and Ralf Stubner's suggestion would hopefully take care of coming back.

要尝试不复制数据，首先让调用使用 load 调用读入的 data，而不是 i；你只需更改这一行。

end <- "1 + (1|cntry/countryyear), data=data, control=glmerControl(optimizer='bobyqa', optCtrl = list(maxfun = 1e9)), family=binomial(link='logit'))"

然后 mclapply 只是运行那些，而不复制数据。

library(parallel)
m.list = mclapply(calls, function(i) eval(parse(text=i)), 
                  mc.cores = 2, mc.preschedule = FALSE)

为了尽量不返回模型中的所有信息（特别是每个模型的完整数据集），在查看 glmer 输出后，我认为最好做任何处理你希望在流程中，而不是修改 glmer 输出，因为修改 glmer 输出可能会使之后更难获得您想要的摘要。这里我只得到摘要，并把它放在一个列表中，所以你也可以很容易地添加其他输出。

library(parallel)
m.list = mclapply(calls, function(i) {
                     a <- eval(parse(text=i))
                     list(summary=summary(a))
                  }, mc.cores = 2, mc.preschedule = FALSE)

请注意，这都是未经测试的...

mclapply 与 lme4 和长向量

mclapply with lme4 and long vectors

parallel-processing

r

lme4

mclapply