Ошибка в RecyclerView при вызове notifyItemRemoved(position)

Ошибка появляется когда удаляю item из 0 позиции, если из других позиций удалять, то все ок. Прогуглил, прочитал что это ошибка в самом RecyclerView. Кто нибудь сталкивался с такой ошибкой? Что можно сделать в общем?

01-13 06:57:16.375 7314-7314/com.example.user.digital E/AndroidRuntime: FATAL EXCEPTION: main
                                                                    Process: com.example.user.digital, PID: 7314
                                                                    java.lang.IndexOutOfBoundsException: Inconsistency detected. Invalid item position 0(offset:-1).state:4 android.support.v7.widget.RecyclerView{47bd973 VFEDH.... .F....ID 0,0-1440,2448 #7f0800ac app:id/recycler_view_for_my_list}, adapter:com.example.user.digital.adapter.MyListAdapter@33b7aaf, layout:android.support.v7.widget.LinearLayoutManager@20ebebc, context:com.example.user.digital.MyListActivity@8f1f8ed
                                                                        at android.support.v7.widget.RecyclerView$Recycler.tryGetViewHolderForPositionByDeadline(RecyclerView.java:5628)
                                                                        at android.support.v7.widget.RecyclerView$Recycler.getViewForPosition(RecyclerView.java:5563)
                                                                        at android.support.v7.widget.RecyclerView$Recycler.getViewForPosition(RecyclerView.java:5559)
                                                                        at android.support.v7.widget.LinearLayoutManager$LayoutState.next(LinearLayoutManager.java:2229)
                                                                        at android.support.v7.widget.LinearLayoutManager.layoutChunk(LinearLayoutManager.java:1556)
                                                                        at android.support.v7.widget.LinearLayoutManager.fill(LinearLayoutManager.java:1516)
                                                                        at android.support.v7.widget.LinearLayoutManager.onLayoutChildren(LinearLayoutManager.java:618)
                                                                        at android.support.v7.widget.RecyclerView.dispatchLayoutStep1(RecyclerView.java:3644)
                                                                        at android.support.v7.widget.RecyclerView.dispatchLayout(RecyclerView.java:3408)
                                                                        at android.support.v7.widget.RecyclerView.consumePendingUpdateOperations(RecyclerView.java:1710)
                                                                        at android.support.v7.widget.RecyclerView$1.run(RecyclerView.java:346)
                                                                        at android.view.Choreographer$CallbackRecord.run(Choreographer.java:874)
                                                                        at android.view.Choreographer.doCallbacks(Choreographer.java:686)
                                                                        at android.view.Choreographer.doFrame(Choreographer.java:618)
                                                                        at android.view.Choreographer$FrameDisplayEventReceiver.run(Choreographer.java:860)
                                                                        at android.os.Handler.handleCallback(Handler.java:751)
                                                                        at android.os.Handler.dispatchMessage(Handler.java:95)
                                                                        at android.os.Looper.loop(Looper.java:154)
                                                                        at android.app.ActivityThread.main(ActivityThread.java:6119)
                                                                        at java.lang.reflect.Method.invoke(Native Method)
                                                                        at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:886)
                                                                        at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:776)

UPD:

public abstract class SwipeControllerActions {
    public void onRightClicked(int position) {
        removeItem(position);
    }
}

В адаптере

private void removeItem(int position) {
    Item item = dataList.get(position);
    MyTask myTask = new MyTask(item);
    myTask.execute();
    dataList.remove(position);
    notifyItemRemoved(position);
}

В ItemTouchHelper

private void setTouchUpListener(final Canvas c, final RecyclerView recyclerView, final RecyclerView.ViewHolder viewHolder, final float dX, final float dY, final int actionState, final boolean isCurrentlyActive) {
        recyclerView.setOnTouchListener(new View.OnTouchListener() {
            @Override
            public boolean onTouch(View v, MotionEvent event) {
                if (event.getAction() == MotionEvent.ACTION_UP) {
                    SwipeController.super.onChildDraw(c, recyclerView, viewHolder, 0F, dY, actionState, isCurrentlyActive);
                    recyclerView.setOnTouchListener(new View.OnTouchListener() {
                        @Override
                        public boolean onTouch(View v, MotionEvent event) {
                            return false;
                        }
                    });
                    setItemsClickable(recyclerView, true);
                    swipeBack = false;
                    if (buttonsActions != null && buttonInstance != null && buttonInstance.contains(event.getX(), event.getY())) {
                        if (buttonShowedState == ButtonsState.RIGHT_VISIBLE) {
                            buttonsActions.onRightClicked(viewHolder.getAdapterPosition());
                        }
                    }
                    buttonShowedState = ButtonsState.GONE;
                    currentItemViewHolder = null;
                }
                return false;
            }
        });
    }