diff --git a/src/e1_input.c b/src/e1_input.c
index a549ba4..ad0778a 100644
--- a/src/e1_input.c
+++ b/src/e1_input.c
@@ -676,10 +676,6 @@
 
 	e1inp_line_get(line);
 
-	/* This line has been already initialized, skip this. */
-	if (line->refcnt > 2)
-		return 0;
-
 	if (line->driver && line->ops && line->driver->line_update) {
 		rc = line->driver->line_update(line);
 	} else
diff --git a/src/input/hsl.c b/src/input/hsl.c
index dc7532b..60eea17 100644
--- a/src/input/hsl.c
+++ b/src/input/hsl.c
@@ -456,9 +456,29 @@
 	return 0;
 }
 
+struct hsl_line {
+	int line_already_initialized;
+};
+
 static int hsl_line_update(struct e1inp_line *line)
 {
 	int ret = -ENOENT;
+	struct hsl_line *hl;
+
+	if (!line->driver_data)
+		line->driver_data = talloc_zero(line, struct hsl_line);
+
+	if (!line->driver_data) {
+		LOGP(DLINP, LOGL_NOTICE, "hsl: OOM in line update\n");
+		return -ENOMEM;
+	}
+	hl = line->driver_data;
+
+	/* We only initialize this line once. */
+	if (hl->line_already_initialized)
+		return 0;
+
+	hl->line_already_initialized = 1;
 
 	switch(line->ops->cfg.ipa.role) {
 	case E1INP_LINE_R_BSC:
diff --git a/src/input/ipaccess.c b/src/input/ipaccess.c
index b7391b3..ea04e8d 100644
--- a/src/input/ipaccess.c
+++ b/src/input/ipaccess.c
@@ -813,9 +813,29 @@
 	return 0;
 }
 
+struct ipaccess_line {
+	int line_already_initialized;
+};
+
 static int ipaccess_line_update(struct e1inp_line *line)
 {
 	int ret = -ENOENT;
+	struct ipaccess_line *il;
+
+	if (!line->driver_data)
+		line->driver_data = talloc_zero(line, struct ipaccess_line);
+
+	if (!line->driver_data) {
+		LOGP(DLINP, LOGL_ERROR, "ipaccess: OOM in line update\n");
+		return -ENOMEM;
+	}
+	il = line->driver_data;
+
+	/* We only initialize this line once. */
+	if (il->line_already_initialized)
+		return 0;
+
+	il->line_already_initialized = 1;
 
 	switch(line->ops->cfg.ipa.role) {
 	case E1INP_LINE_R_BSC: {